Code move from internal SVN
git-svn-id: https://hyracks.googlecode.com/svn/trunk/hyracks@2 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-core/src/main/assembly/binary-assembly.xml b/hyracks-core/src/main/assembly/binary-assembly.xml
new file mode 100644
index 0000000..cd598d9
--- /dev/null
+++ b/hyracks-core/src/main/assembly/binary-assembly.xml
@@ -0,0 +1,23 @@
+<assembly>
+ <id>binary-assembly</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>target/appassembler/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>target/appassembler/lib</directory>
+ <outputDirectory>lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>docs</directory>
+ <outputDirectory>docs</outputDirectory>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/hyracks-core/src/main/assembly/source-assembly.xml b/hyracks-core/src/main/assembly/source-assembly.xml
new file mode 100644
index 0000000..82db140
--- /dev/null
+++ b/hyracks-core/src/main/assembly/source-assembly.xml
@@ -0,0 +1,17 @@
+<assembly>
+ <id>source-assembly</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>.</directory>
+ <outputDirectory>hyracks-core</outputDirectory>
+ <excludes>
+ <exclude>target/**</exclude>
+ </excludes>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/Endpoint.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/Endpoint.java
new file mode 100644
index 0000000..c0d23c1
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/Endpoint.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.comm;
+
+import java.io.Serializable;
+import java.util.UUID;
+
+public final class Endpoint implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final UUID id;
+
+ private final NetworkAddress address;
+
+ private final int receiverIndex;
+
+ public Endpoint(NetworkAddress address, int receiverIndex) throws Exception {
+ id = UUID.randomUUID();
+ this.address = address;
+ this.receiverIndex = receiverIndex;
+ }
+
+ public UUID getEndpointId() {
+ return id;
+ }
+
+ public NetworkAddress getNetworkAddress() {
+ return address;
+ }
+
+ public int getReceiverIndex() {
+ return receiverIndex;
+ }
+
+ @Override
+ public int hashCode() {
+ return id.hashCode() + address.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof Endpoint)) {
+ return false;
+ }
+ Endpoint oe = (Endpoint) o;
+ return oe.id.equals(id) && oe.address.equals(address);
+ }
+
+ @Override
+ public String toString() {
+ return "[" + address + ":" + id + "]";
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/FrameConstants.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/FrameConstants.java
new file mode 100644
index 0000000..cc76bd4
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/FrameConstants.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.comm;
+
+public interface FrameConstants {
+ public static final int SIZE_LEN = 4;
+
+ public static final boolean DEBUG_FRAME_IO = false;
+
+ public static final int FRAME_FIELD_MAGIC = 0x12345678;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IConnectionDemultiplexer.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IConnectionDemultiplexer.java
new file mode 100644
index 0000000..0587722
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IConnectionDemultiplexer.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.comm;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface IConnectionDemultiplexer {
+ public int getSenderCount();
+
+ public IConnectionEntry findNextReadyEntry(int lastReadSender);
+
+ public void unreadyEntry(int index);
+
+ public int closeEntry(int index) throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IConnectionEntry.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IConnectionEntry.java
new file mode 100644
index 0000000..125c1aa
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IConnectionEntry.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.comm;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+
+public interface IConnectionEntry {
+ ByteBuffer getReadBuffer();
+
+ SelectionKey getSelectionKey();
+
+ void setDataReceiveListener(IDataReceiveListener listener);
+
+ void attach(Object attachment);
+
+ Object getAttachment();
+
+ void close() throws IOException;
+
+ void write(ByteBuffer buffer);
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IDataReceiveListener.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IDataReceiveListener.java
new file mode 100644
index 0000000..463e4e1
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IDataReceiveListener.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.comm;
+
+import java.io.IOException;
+
+public interface IDataReceiveListener {
+ public void dataReceived(IConnectionEntry entry) throws IOException;
+
+ public void eos(IConnectionEntry entry);
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IDataReceiveListenerFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IDataReceiveListenerFactory.java
new file mode 100644
index 0000000..91f921c
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IDataReceiveListenerFactory.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.comm;
+
+import java.util.UUID;
+
+public interface IDataReceiveListenerFactory {
+ public IDataReceiveListener getDataReceiveListener(UUID endpointUUID, IConnectionEntry entry, int senderIndex);
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IFrameReader.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IFrameReader.java
new file mode 100644
index 0000000..0188578
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IFrameReader.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.comm;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface IFrameReader {
+ public void open() throws HyracksDataException;
+
+ public boolean nextFrame(ByteBuffer buffer) throws HyracksDataException;
+
+ public void close() throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IFrameWriter.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IFrameWriter.java
new file mode 100644
index 0000000..eccd6cb
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/IFrameWriter.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.comm;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface IFrameWriter {
+ public void open() throws HyracksDataException;
+
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException;
+
+ public void close() throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/NetworkAddress.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/NetworkAddress.java
new file mode 100644
index 0000000..868221d
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/comm/NetworkAddress.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.comm;
+
+import java.io.Serializable;
+import java.net.InetAddress;
+
+public final class NetworkAddress implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final InetAddress ipAddress;
+
+ private final int port;
+
+ public NetworkAddress(InetAddress ipAddress, int port) {
+ this.ipAddress = ipAddress;
+ this.port = port;
+ }
+
+ public InetAddress getIpAddress() {
+ return ipAddress;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ @Override
+ public String toString() {
+ return ipAddress + ":" + port;
+ }
+
+ @Override
+ public int hashCode() {
+ return ipAddress.hashCode() + port;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof NetworkAddress)) {
+ return false;
+ }
+ NetworkAddress on = (NetworkAddress) o;
+ return on.port == port && on.ipAddress.equals(ipAddress);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/constraints/AbsoluteLocationConstraint.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/constraints/AbsoluteLocationConstraint.java
new file mode 100644
index 0000000..858bb65
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/constraints/AbsoluteLocationConstraint.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.constraints;
+
+public class AbsoluteLocationConstraint extends LocationConstraint {
+ private static final long serialVersionUID = 1L;
+
+ private String nodeId;
+
+ public AbsoluteLocationConstraint() {
+ }
+
+ public AbsoluteLocationConstraint(String nodeId) {
+ this.nodeId = nodeId;
+ }
+
+ @Override
+ public ConstraintType getConstraintType() {
+ return ConstraintType.ABSOLUTE;
+ }
+
+ public String getLocationId() {
+ return nodeId;
+ }
+
+ public void setLocationId(String nodeId) {
+ this.nodeId = nodeId;
+ }
+
+ public String toString() {
+ return getConstraintType() + ":" + nodeId;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/constraints/LocationConstraint.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/constraints/LocationConstraint.java
new file mode 100644
index 0000000..e88a1b2
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/constraints/LocationConstraint.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.constraints;
+
+import java.io.Serializable;
+
+public abstract class LocationConstraint implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ public enum ConstraintType {
+ ABSOLUTE,
+ }
+
+ public abstract ConstraintType getConstraintType();
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/constraints/PartitionConstraint.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/constraints/PartitionConstraint.java
new file mode 100644
index 0000000..c825539
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/constraints/PartitionConstraint.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.constraints;
+
+import java.io.Serializable;
+import java.util.Arrays;
+
+public class PartitionConstraint implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final LocationConstraint[] locationConstraints;
+
+ public PartitionConstraint(LocationConstraint[] locationConstraints) {
+ this.locationConstraints = locationConstraints;
+ }
+
+ public LocationConstraint[] getLocationConstraints() {
+ return locationConstraints;
+ }
+
+ public String toString() {
+ return Arrays.deepToString(locationConstraints);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/controller/IClusterController.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/controller/IClusterController.java
new file mode 100644
index 0000000..af86093
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/controller/IClusterController.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.controller;
+
+import java.rmi.Remote;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.api.job.statistics.JobStatistics;
+import edu.uci.ics.hyracks.api.job.statistics.StageletStatistics;
+
+public interface IClusterController extends Remote {
+ public void registerNode(INodeController nodeController) throws Exception;
+
+ public void unregisterNode(INodeController nodeController) throws Exception;
+
+ public INodeController lookupNode(String id) throws Exception;
+
+ public void notifyStageletComplete(UUID jobId, UUID stageId, String nodeId, StageletStatistics statistics)
+ throws Exception;
+
+ /*
+ * Client Job Control methods.
+ */
+ public UUID createJob(JobSpecification jobSpec) throws Exception;
+
+ public UUID createJob(JobSpecification jobSpec, EnumSet<JobFlag> jobFlags) throws Exception;
+
+ public JobStatus getJobStatus(UUID jobId) throws Exception;
+
+ public void start(UUID jobId) throws Exception;
+
+ public JobStatistics waitForCompletion(UUID jobId) throws Exception;
+
+ public Map<String,INodeController> getRegistry() throws Exception;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/controller/INodeController.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/controller/INodeController.java
new file mode 100644
index 0000000..c271356
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/controller/INodeController.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.controller;
+
+import java.rmi.Remote;
+import java.util.Map;
+import java.util.UUID;
+
+import edu.uci.ics.hyracks.api.comm.Endpoint;
+import edu.uci.ics.hyracks.api.dataflow.PortInstanceId;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobStage;
+import edu.uci.ics.hyracks.config.NCConfig;
+
+public interface INodeController extends Remote {
+ public String getId() throws Exception;
+
+ public NCConfig getConfiguration() throws Exception;
+
+ public NodeCapability getNodeCapability() throws Exception;
+
+ public Map<PortInstanceId, Endpoint> initializeJobletPhase1(UUID jobId, JobPlan plan, JobStage stage)
+ throws Exception;
+
+ public void initializeJobletPhase2(UUID jobId, JobPlan plan, JobStage stage,
+ Map<PortInstanceId, Endpoint> globalPortMap) throws Exception;
+
+ public void commitJobletInitialization(UUID jobId, JobPlan plan, JobStage stage) throws Exception;
+
+ public void cleanUpJob(UUID jobId) throws Exception;
+
+ public void startStage(UUID jobId, UUID stageId) throws Exception;
+
+ public void notifyRegistration(IClusterController ccs) throws Exception;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/controller/NodeCapability.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/controller/NodeCapability.java
new file mode 100644
index 0000000..62d627c
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/controller/NodeCapability.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.controller;
+
+import java.io.Serializable;
+
+public class NodeCapability implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private int cpuCount;
+
+ public int getCPUCount() {
+ return cpuCount;
+ }
+
+ public void setCPUCount(int cpuCount) {
+ this.cpuCount = cpuCount;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/ActivityNodeId.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/ActivityNodeId.java
new file mode 100644
index 0000000..12f76fe
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/ActivityNodeId.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow;
+
+import java.io.Serializable;
+import java.util.UUID;
+
+public final class ActivityNodeId implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private final OperatorDescriptorId odId;
+ private final UUID id;
+
+ public ActivityNodeId(OperatorDescriptorId odId, UUID id) {
+ this.odId = odId;
+ this.id = id;
+ }
+
+ public OperatorDescriptorId getOperatorDescriptorId() {
+ return odId;
+ }
+
+ public UUID getLocalId() {
+ return id;
+ }
+
+ @Override
+ public int hashCode() {
+ return odId.hashCode() + id.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof ActivityNodeId)) {
+ return false;
+ }
+ ActivityNodeId other = (ActivityNodeId) o;
+ return other.odId.equals(odId) && other.id.equals(id);
+ }
+
+ public String toString() {
+ return "ANID:" + id;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/ConnectorDescriptorId.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/ConnectorDescriptorId.java
new file mode 100644
index 0000000..c7f3175
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/ConnectorDescriptorId.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow;
+
+import java.io.Serializable;
+import java.util.UUID;
+
+public final class ConnectorDescriptorId implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private UUID id;
+
+ public ConnectorDescriptorId(UUID id) {
+ this.id = id;
+ }
+
+ public UUID getId() {
+ return id;
+ }
+
+ @Override
+ public int hashCode() {
+ return id.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof ConnectorDescriptorId)) {
+ return false;
+ }
+ ConnectorDescriptorId other = (ConnectorDescriptorId) obj;
+ return id.equals(other.id);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IActivityGraphBuilder.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IActivityGraphBuilder.java
new file mode 100644
index 0000000..c6bc5e3
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IActivityGraphBuilder.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow;
+
+public interface IActivityGraphBuilder {
+ public void addTask(IActivityNode task);
+
+ public void addBlockingEdge(IActivityNode blocker, IActivityNode blocked);
+
+ public void addSourceEdge(int operatorInputIndex, IActivityNode task, int taskInputIndex);
+
+ public void addTargetEdge(int operatorOutputIndex, IActivityNode task, int taskOutputIndex);
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IActivityNode.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IActivityNode.java
new file mode 100644
index 0000000..5b57ee3
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IActivityNode.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public interface IActivityNode extends Serializable {
+ public ActivityNodeId getActivityNodeId();
+
+ public IOperatorDescriptor getOwner();
+
+ public boolean supportsPushInterface();
+
+ public boolean supportsPullInterface();
+
+ public IOperatorNodePushable createPushRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition);
+
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition);
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IConnectorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IConnectorDescriptor.java
new file mode 100644
index 0000000..600fd41
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IConnectorDescriptor.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow;
+
+import java.io.Serializable;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.api.comm.IConnectionDemultiplexer;
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+/**
+ * Connector that connects operators in a Job.
+ *
+ * @author vinayakb
+ */
+public interface IConnectorDescriptor extends Serializable {
+ /**
+ * Gets the id of the connector.
+ *
+ * @return
+ */
+ public ConnectorDescriptorId getConnectorId();
+
+ /**
+ * Factory method to create the send side writer that writes into this connector.
+ *
+ * @param ctx
+ * Context
+ * @param plan
+ * Job plan
+ * @param edwFactory
+ * Endpoint writer factory.
+ * @param index
+ * ordinal index of the data producer partition.
+ * @return data writer.
+ * @throws Exception
+ */
+ public IFrameWriter createSendSideWriter(HyracksContext ctx, JobPlan plan, IEndpointDataWriterFactory edwFactory,
+ int index) throws HyracksDataException;
+
+ /**
+ * Factory metod to create the receive side reader that reads data from this connector.
+ *
+ * @param ctx
+ * Context
+ * @param plan
+ * Job plan
+ * @param demux
+ * Connection Demultiplexer
+ * @param index
+ * ordinal index of the data consumer partition
+ * @return data reader
+ * @throws HyracksDataException
+ */
+ public IFrameReader createReceiveSideReader(HyracksContext ctx, JobPlan plan, IConnectionDemultiplexer demux,
+ int index) throws HyracksDataException;
+
+ /**
+ * Translate this connector descriptor to JSON.
+ *
+ * @return
+ * @throws JSONException
+ */
+ public JSONObject toJSON() throws JSONException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IDataReader.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IDataReader.java
new file mode 100644
index 0000000..b5f45cb
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IDataReader.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Provides data to a consumer.
+ *
+ * @author vinayakb
+ */
+public interface IDataReader<T> {
+ /**
+ * Reads one data item.
+ *
+ * @return data. <code>null</code> indicates end of stream.
+ * @throws Exception
+ */
+ public T readData() throws HyracksDataException;
+
+ /**
+ * Closes this reader.
+ *
+ * @throws Exception
+ */
+ public void close() throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IDataWriter.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IDataWriter.java
new file mode 100644
index 0000000..128576c
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IDataWriter.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Accepts data from data producers.
+ *
+ * @author vinayakb
+ */
+public interface IDataWriter<T> {
+ /**
+ * Pushes data to the acceptor.
+ *
+ * @param data
+ * - Data pushed to the acceptor. <code>null</code> indicates the end of stream.
+ * @throws HyracksDataException
+ */
+ public void writeData(T data) throws HyracksDataException;
+
+ /**
+ * Closes this writer.
+ *
+ * @throws Exception
+ */
+ public void close() throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IEndpointDataWriterFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IEndpointDataWriterFactory.java
new file mode 100644
index 0000000..45c589a
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IEndpointDataWriterFactory.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface IEndpointDataWriterFactory {
+ public IFrameWriter createFrameWriter(int index) throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOpenableDataReader.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOpenableDataReader.java
new file mode 100644
index 0000000..d0b6474
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOpenableDataReader.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface IOpenableDataReader<T> extends IDataReader<T> {
+ public void open() throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOpenableDataWriter.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOpenableDataWriter.java
new file mode 100644
index 0000000..b2b15c6
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOpenableDataWriter.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface IOpenableDataWriter<T> extends IDataWriter<T> {
+ public void open() throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorDescriptor.java
new file mode 100644
index 0000000..17a0fc5
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorDescriptor.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow;
+
+import java.io.Serializable;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.api.constraints.PartitionConstraint;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+/**
+ * Descriptor for operators in Hyracks.
+ *
+ * @author vinayakb
+ */
+public interface IOperatorDescriptor extends Serializable {
+ /**
+ * Returns the id of the operator.
+ *
+ * @return operator id
+ */
+ public OperatorDescriptorId getOperatorId();
+
+ /**
+ * Returns the number of inputs into this operator.
+ *
+ * @return Number of inputs.
+ */
+ public int getInputArity();
+
+ /**
+ * Returns the number of outputs out of this operator.
+ *
+ * @return Number of outputs.
+ */
+ public int getOutputArity();
+
+ /**
+ * Returns the partition constraint requested for scheduling this operator. This value is set as part of Job creation by the client.
+ *
+ * @return the partition constraint.
+ */
+ public PartitionConstraint getPartitionConstraint();
+
+ /**
+ * Sets the partition constraint.
+ *
+ * @param partitionConstraint
+ * - partition constraint.
+ */
+ public void setPartitionConstraint(PartitionConstraint partitionConstraint);
+
+ /**
+ * Returns the final partition locations selected for scheduling. These are decided by Hyracks such that they satisfy the partition constraints.
+ *
+ * @return array indicating number and node ids of the nodes to schedule the operator runtimes.
+ */
+ public String[] getPartitions();
+
+ /**
+ * Sets the partition locations.
+ *
+ * @param partitions
+ * node ids to schedule the operators.
+ */
+ public void setPartitions(String[] partitions);
+
+ /**
+ * Gets the output record descriptor
+ *
+ * @return Array of RecordDescriptor, one per output.
+ */
+ public RecordDescriptor[] getOutputRecordDescriptors();
+
+ /**
+ * Contributes the activity graph that describes the behavior of this operator.
+ *
+ * @param builder
+ * - graph builder
+ */
+ public void contributeTaskGraph(IActivityGraphBuilder builder);
+
+ /**
+ * Translates this operator descriptor to JSON.
+ */
+ public JSONObject toJSON() throws JSONException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorNodePullable.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorNodePullable.java
new file mode 100644
index 0000000..1fa039f
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorNodePullable.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow;
+
+public interface IOperatorNodePullable extends IOpenableDataReader<Object[]> {
+ public void setDataReader(int index, IOpenableDataWriter<Object[]> writer);
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorNodePushable.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorNodePushable.java
new file mode 100644
index 0000000..13ac15c
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorNodePushable.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+
+public interface IOperatorNodePushable extends IFrameWriter {
+ public void setFrameWriter(int index, IFrameWriter writer);
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/OperatorDescriptorId.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/OperatorDescriptorId.java
new file mode 100644
index 0000000..bbf3cf5
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/OperatorDescriptorId.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow;
+
+import java.io.Serializable;
+import java.util.UUID;
+
+public final class OperatorDescriptorId implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final UUID id;
+
+ public OperatorDescriptorId(UUID id) {
+ this.id = id;
+ }
+
+ public UUID getId() {
+ return id;
+ }
+
+ @Override
+ public int hashCode() {
+ return id.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ if (!(o instanceof OperatorDescriptorId)) {
+ return false;
+ }
+ return ((OperatorDescriptorId) o).id.equals(id);
+ }
+
+ @Override
+ public String toString() {
+ return "ODID:" + id;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/OperatorInstanceId.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/OperatorInstanceId.java
new file mode 100644
index 0000000..f6ee39b
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/OperatorInstanceId.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow;
+
+import java.io.Serializable;
+
+public final class OperatorInstanceId implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private OperatorDescriptorId odId;
+ private int partition;
+
+ public OperatorInstanceId(OperatorDescriptorId odId, int partition) {
+ this.odId = odId;
+ this.partition = partition;
+ }
+
+ public OperatorDescriptorId getOperatorId() {
+ return odId;
+ }
+
+ public int getPartition() {
+ return partition;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((odId == null) ? 0 : odId.hashCode());
+ result = prime * result + partition;
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ OperatorInstanceId other = (OperatorInstanceId) obj;
+ if (odId == null) {
+ if (other.odId != null)
+ return false;
+ } else if (!odId.equals(other.odId))
+ return false;
+ if (partition != other.partition)
+ return false;
+ return true;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/PortInstanceId.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/PortInstanceId.java
new file mode 100644
index 0000000..d8057d9
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/PortInstanceId.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow;
+
+import java.io.Serializable;
+
+public final class PortInstanceId implements Serializable {
+ public enum Direction {
+ INPUT,
+ OUTPUT,
+ }
+
+ private static final long serialVersionUID = 1L;
+
+ private OperatorDescriptorId odId;
+ private Direction direction;
+ private int portIndex;
+ private int partition;
+
+ public PortInstanceId(OperatorDescriptorId odId, Direction direction, int portIndex, int partition) {
+ this.odId = odId;
+ this.direction = direction;
+ this.portIndex = portIndex;
+ this.partition = partition;
+ }
+
+ public OperatorDescriptorId getOperatorId() {
+ return odId;
+ }
+
+ public Direction getDirection() {
+ return direction;
+ }
+
+ public int getPortIndex() {
+ return portIndex;
+ }
+
+ public int getPartition() {
+ return partition;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((direction == null) ? 0 : direction.hashCode());
+ result = prime * result + ((odId == null) ? 0 : odId.hashCode());
+ result = prime * result + partition;
+ result = prime * result + portIndex;
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ PortInstanceId other = (PortInstanceId) obj;
+ if (direction == null) {
+ if (other.direction != null)
+ return false;
+ } else if (!direction.equals(other.direction))
+ return false;
+ if (odId == null) {
+ if (other.odId != null)
+ return false;
+ } else if (!odId.equals(other.odId))
+ return false;
+ if (partition != other.partition)
+ return false;
+ if (portIndex != other.portIndex)
+ return false;
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return odId + ":" + direction + ":" + partition + ":" + portIndex;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryComparator.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryComparator.java
new file mode 100644
index 0000000..767684d
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryComparator.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow.value;
+
+public interface IBinaryComparator {
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2);
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryComparatorFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryComparatorFactory.java
new file mode 100644
index 0000000..9976818
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryComparatorFactory.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow.value;
+
+import java.io.Serializable;
+
+public interface IBinaryComparatorFactory extends Serializable {
+ public IBinaryComparator createBinaryComparator();
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryHashFunction.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryHashFunction.java
new file mode 100644
index 0000000..a995703
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryHashFunction.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow.value;
+
+public interface IBinaryHashFunction {
+ int hash(byte[] bytes, int offset, int length);
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryHashFunctionFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryHashFunctionFactory.java
new file mode 100644
index 0000000..a064d5f
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryHashFunctionFactory.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow.value;
+
+import java.io.Serializable;
+
+public interface IBinaryHashFunctionFactory extends Serializable {
+ public IBinaryHashFunction createBinaryHashFunction();
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IComparator.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IComparator.java
new file mode 100644
index 0000000..2d24496
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IComparator.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow.value;
+
+public interface IComparator<T> {
+ public int compare(T o1, T o2);
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IComparatorFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IComparatorFactory.java
new file mode 100644
index 0000000..ef9161f
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IComparatorFactory.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow.value;
+
+import java.io.Serializable;
+
+public interface IComparatorFactory<T> extends Serializable {
+ public IComparator<T> createComparator();
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IHashFunction.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IHashFunction.java
new file mode 100644
index 0000000..9124fca
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IHashFunction.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow.value;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface IHashFunction<T> {
+ public int hash(T o) throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IHashFunctionFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IHashFunctionFactory.java
new file mode 100644
index 0000000..b67a96c
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IHashFunctionFactory.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow.value;
+
+import java.io.Serializable;
+
+public interface IHashFunctionFactory<T> extends Serializable {
+ public IHashFunction<T> createHashFunction();
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ISerializerDeserializer.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ISerializerDeserializer.java
new file mode 100644
index 0000000..33ba602
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ISerializerDeserializer.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow.value;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface ISerializerDeserializer<T> extends Serializable {
+ /**
+ * Deserialization method.
+ *
+ * @param in
+ * - Stream to read instance from.
+ * @return A new instance of T with data.
+ */
+ public T deserialize(DataInput in) throws HyracksDataException;
+
+ /**
+ * Serialization method.
+ *
+ * @param instance
+ * - the instance to serialize.
+ * @param out
+ * - Stream to write data to.
+ */
+ public void serialize(T instance, DataOutput out) throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITuplePartitionComputer.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITuplePartitionComputer.java
new file mode 100644
index 0000000..d9b023d
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITuplePartitionComputer.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow.value;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAccessor;
+
+public interface ITuplePartitionComputer {
+ public int partition(FrameTupleAccessor accessor, int tIndex, int nParts) throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITuplePartitionComputerFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITuplePartitionComputerFactory.java
new file mode 100644
index 0000000..eb2bbe8
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITuplePartitionComputerFactory.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow.value;
+
+import java.io.Serializable;
+
+public interface ITuplePartitionComputerFactory extends Serializable {
+ public ITuplePartitionComputer createPartitioner();
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/RecordDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/RecordDescriptor.java
new file mode 100644
index 0000000..20dc034
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/RecordDescriptor.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow.value;
+
+import java.io.Serializable;
+
+@SuppressWarnings("unchecked")
+public final class RecordDescriptor implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final ISerializerDeserializer[] fields;
+
+ public RecordDescriptor(ISerializerDeserializer[] fields) {
+ this.fields = fields;
+ }
+
+ public ISerializerDeserializer[] getFields() {
+ return fields;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/exceptions/HyracksDataException.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/exceptions/HyracksDataException.java
new file mode 100644
index 0000000..1bdfe4b
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/exceptions/HyracksDataException.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.exceptions;
+
+import java.io.IOException;
+
+public class HyracksDataException extends IOException {
+ private static final long serialVersionUID = 1L;
+
+ public HyracksDataException() {
+ }
+
+ public HyracksDataException(String message) {
+ super(message);
+ }
+
+ public HyracksDataException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public HyracksDataException(Throwable cause) {
+ super(cause);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/IOperatorEnvironment.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/IOperatorEnvironment.java
new file mode 100644
index 0000000..b41cb35
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/IOperatorEnvironment.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.job;
+
+public interface IOperatorEnvironment {
+ public void set(String name, Object value);
+
+ public Object get(String name);
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/JobFlag.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/JobFlag.java
new file mode 100644
index 0000000..9219ec3
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/JobFlag.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.job;
+
+public enum JobFlag {
+ COLLECT_FRAME_COUNTS
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/JobPlan.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/JobPlan.java
new file mode 100644
index 0000000..8c71438
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/JobPlan.java
@@ -0,0 +1,164 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.job;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.api.dataflow.ActivityNodeId;
+import edu.uci.ics.hyracks.api.dataflow.IActivityNode;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class JobPlan implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final JobSpecification jobSpec;
+
+ private final EnumSet<JobFlag> jobFlags;
+
+ private final Map<ActivityNodeId, IActivityNode> activityNodes;
+
+ private final Map<ActivityNodeId, Set<ActivityNodeId>> blocker2blockedMap;
+
+ private final Map<ActivityNodeId, Set<ActivityNodeId>> blocked2blockerMap;
+
+ private final Map<OperatorDescriptorId, Set<ActivityNodeId>> operatorTaskMap;
+
+ private final Map<ActivityNodeId, List<Integer>> taskInputMap;
+
+ private final Map<ActivityNodeId, List<Integer>> taskOutputMap;
+
+ private final Map<OperatorDescriptorId, List<ActivityNodeId>> operatorInputMap;
+
+ private final Map<OperatorDescriptorId, List<ActivityNodeId>> operatorOutputMap;
+
+ private JobStage endStage;
+
+ public JobPlan(JobSpecification jobSpec, EnumSet<JobFlag> jobFlags) {
+ this.jobSpec = jobSpec;
+ this.jobFlags = jobFlags;
+ activityNodes = new HashMap<ActivityNodeId, IActivityNode>();
+ blocker2blockedMap = new HashMap<ActivityNodeId, Set<ActivityNodeId>>();
+ blocked2blockerMap = new HashMap<ActivityNodeId, Set<ActivityNodeId>>();
+ operatorTaskMap = new HashMap<OperatorDescriptorId, Set<ActivityNodeId>>();
+ taskInputMap = new HashMap<ActivityNodeId, List<Integer>>();
+ taskOutputMap = new HashMap<ActivityNodeId, List<Integer>>();
+ operatorInputMap = new HashMap<OperatorDescriptorId, List<ActivityNodeId>>();
+ operatorOutputMap = new HashMap<OperatorDescriptorId, List<ActivityNodeId>>();
+ }
+
+ public JobSpecification getJobSpecification() {
+ return jobSpec;
+ }
+
+ public EnumSet<JobFlag> getJobFlags() {
+ return jobFlags;
+ }
+
+ public Map<ActivityNodeId, IActivityNode> getActivityNodeMap() {
+ return activityNodes;
+ }
+
+ public Map<ActivityNodeId, Set<ActivityNodeId>> getBlocker2BlockedMap() {
+ return blocker2blockedMap;
+ }
+
+ public Map<ActivityNodeId, Set<ActivityNodeId>> getBlocked2BlockerMap() {
+ return blocked2blockerMap;
+ }
+
+ public Map<OperatorDescriptorId, Set<ActivityNodeId>> getOperatorTaskMap() {
+ return operatorTaskMap;
+ }
+
+ public Map<ActivityNodeId, List<Integer>> getTaskInputMap() {
+ return taskInputMap;
+ }
+
+ public Map<ActivityNodeId, List<Integer>> getTaskOutputMap() {
+ return taskOutputMap;
+ }
+
+ public Map<OperatorDescriptorId, List<ActivityNodeId>> getOperatorInputMap() {
+ return operatorInputMap;
+ }
+
+ public Map<OperatorDescriptorId, List<ActivityNodeId>> getOperatorOutputMap() {
+ return operatorOutputMap;
+ }
+
+ public void setEndStage(JobStage endStage) {
+ this.endStage = endStage;
+ }
+
+ public JobStage getEndStage() {
+ return endStage;
+ }
+
+ public List<IConnectorDescriptor> getTaskInputs(ActivityNodeId hanId) {
+ List<Integer> inputIndexes = taskInputMap.get(hanId);
+ if (inputIndexes == null) {
+ return null;
+ }
+ OperatorDescriptorId ownerId = hanId.getOperatorDescriptorId();
+ List<IConnectorDescriptor> inputs = new ArrayList<IConnectorDescriptor>();
+ for (Integer i : inputIndexes) {
+ inputs.add(jobSpec.getInputConnectorDescriptor(ownerId, i));
+ }
+ return inputs;
+ }
+
+ public List<IConnectorDescriptor> getTaskOutputs(ActivityNodeId hanId) {
+ List<Integer> outputIndexes = taskOutputMap.get(hanId);
+ if (outputIndexes == null) {
+ return null;
+ }
+ OperatorDescriptorId ownerId = hanId.getOperatorDescriptorId();
+ List<IConnectorDescriptor> outputs = new ArrayList<IConnectorDescriptor>();
+ for (Integer i : outputIndexes) {
+ outputs.add(jobSpec.getOutputConnectorDescriptor(ownerId, i));
+ }
+ return outputs;
+ }
+
+ public RecordDescriptor getTaskInputRecordDescriptor(ActivityNodeId hanId, int inputIndex) {
+ int opInputIndex = getTaskInputMap().get(hanId).get(inputIndex);
+ return jobSpec.getOperatorInputRecordDescriptor(hanId.getOperatorDescriptorId(), opInputIndex);
+ }
+
+ public RecordDescriptor getTaskOutputRecordDescriptor(ActivityNodeId hanId, int outputIndex) {
+ int opOutputIndex = getTaskOutputMap().get(hanId).get(outputIndex);
+ return jobSpec.getOperatorOutputRecordDescriptor(hanId.getOperatorDescriptorId(), opOutputIndex);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buffer = new StringBuilder();
+ buffer.append("ActivityNodes: " + activityNodes);
+ buffer.append('\n');
+ buffer.append("Blocker->Blocked: " + blocker2blockedMap);
+ buffer.append('\n');
+ buffer.append("Blocked->Blocker: " + blocked2blockerMap);
+ buffer.append('\n');
+ return buffer.toString();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java
new file mode 100644
index 0000000..27bbdc8
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java
@@ -0,0 +1,228 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.job;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.util.Pair;
+
+public class JobSpecification implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final List<OperatorDescriptorId> roots;
+
+ private final Map<OperatorDescriptorId, IOperatorDescriptor> opMap;
+
+ private final Map<ConnectorDescriptorId, IConnectorDescriptor> connMap;
+
+ private final Map<OperatorDescriptorId, List<IConnectorDescriptor>> opInputMap;
+
+ private final Map<OperatorDescriptorId, List<IConnectorDescriptor>> opOutputMap;
+
+ private final Map<ConnectorDescriptorId, Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>>> connectorOpMap;
+
+ public JobSpecification() {
+ roots = new ArrayList<OperatorDescriptorId>();
+ opMap = new HashMap<OperatorDescriptorId, IOperatorDescriptor>();
+ connMap = new HashMap<ConnectorDescriptorId, IConnectorDescriptor>();
+ opInputMap = new HashMap<OperatorDescriptorId, List<IConnectorDescriptor>>();
+ opOutputMap = new HashMap<OperatorDescriptorId, List<IConnectorDescriptor>>();
+ connectorOpMap = new HashMap<ConnectorDescriptorId, Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>>>();
+ }
+
+ public void addRoot(IOperatorDescriptor op) {
+ roots.add(op.getOperatorId());
+ }
+
+ public void connect(IConnectorDescriptor conn, IOperatorDescriptor producerOp, int producerPort,
+ IOperatorDescriptor consumerOp, int consumerPort) {
+ insertIntoIndexedMap(opInputMap, consumerOp.getOperatorId(), consumerPort, conn);
+ insertIntoIndexedMap(opOutputMap, producerOp.getOperatorId(), producerPort, conn);
+ connectorOpMap.put(conn.getConnectorId(),
+ new Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>>(
+ new Pair<IOperatorDescriptor, Integer>(producerOp, producerPort),
+ new Pair<IOperatorDescriptor, Integer>(consumerOp, consumerPort)));
+ }
+
+ private <T> void extend(List<T> list, int index) {
+ int n = list.size();
+ for (int i = n; i <= index; ++i) {
+ list.add(null);
+ }
+ }
+
+ public Map<ConnectorDescriptorId, IConnectorDescriptor> getConnectorMap() {
+ return connMap;
+ }
+
+ public Map<ConnectorDescriptorId, Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>>> getConnectorOperatorMap() {
+ return connectorOpMap;
+ }
+
+ public RecordDescriptor getConnectorRecordDescriptor(IConnectorDescriptor conn) {
+ Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
+ .getConnectorId());
+ return connInfo.first.first.getOutputRecordDescriptors()[connInfo.first.second];
+ }
+
+ public IOperatorDescriptor getConsumer(IConnectorDescriptor conn) {
+ Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
+ .getConnectorId());
+ return connInfo.second.first;
+ }
+
+ public int getConsumerInputIndex(IConnectorDescriptor conn) {
+ Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
+ .getConnectorId());
+ return connInfo.second.second;
+ }
+
+ public IConnectorDescriptor getInputConnectorDescriptor(IOperatorDescriptor op, int inputIndex) {
+ return getInputConnectorDescriptor(op.getOperatorId(), inputIndex);
+ }
+
+ public IConnectorDescriptor getInputConnectorDescriptor(OperatorDescriptorId odId, int inputIndex) {
+ return opInputMap.get(odId).get(inputIndex);
+ }
+
+ public Map<OperatorDescriptorId, List<IConnectorDescriptor>> getOperatorInputMap() {
+ return opInputMap;
+ }
+
+ public RecordDescriptor getOperatorInputRecordDescriptor(OperatorDescriptorId odId, int inputIndex) {
+ return getConnectorRecordDescriptor(getInputConnectorDescriptor(odId, inputIndex));
+ }
+
+ public Map<OperatorDescriptorId, IOperatorDescriptor> getOperatorMap() {
+ return opMap;
+ }
+
+ public Map<OperatorDescriptorId, List<IConnectorDescriptor>> getOperatorOutputMap() {
+ return opOutputMap;
+ }
+
+ public RecordDescriptor getOperatorOutputRecordDescriptor(OperatorDescriptorId odId, int outputIndex) {
+ return getConnectorRecordDescriptor(getOutputConnectorDescriptor(odId, outputIndex));
+ }
+
+ public IConnectorDescriptor getOutputConnectorDescriptor(IOperatorDescriptor op, int outputIndex) {
+ return getOutputConnectorDescriptor(op.getOperatorId(), outputIndex);
+ }
+
+ public IConnectorDescriptor getOutputConnectorDescriptor(OperatorDescriptorId odId, int outputIndex) {
+ return opOutputMap.get(odId).get(outputIndex);
+ }
+
+ public IOperatorDescriptor getProducer(IConnectorDescriptor conn) {
+ Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
+ .getConnectorId());
+ return connInfo.first.first;
+ }
+
+ public int getProducerOutputIndex(IConnectorDescriptor conn) {
+ Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
+ .getConnectorId());
+ return connInfo.first.second;
+ }
+
+ public List<OperatorDescriptorId> getRoots() {
+ return roots;
+ }
+
+ private <K, V> void insertIntoIndexedMap(Map<K, List<V>> map, K key, int index, V value) {
+ List<V> vList = map.get(key);
+ if (vList == null) {
+ vList = new ArrayList<V>();
+ map.put(key, vList);
+ }
+ extend(vList, index);
+ vList.set(index, value);
+ }
+
+ public String toString() {
+ StringBuilder buffer = new StringBuilder();
+
+ for (Map.Entry<OperatorDescriptorId, IOperatorDescriptor> e : opMap.entrySet()) {
+ buffer.append(e.getKey().getId()).append(" : ").append(e.getValue().toString()).append("\n");
+ IOperatorDescriptor op = e.getValue();
+ buffer.append(" Partition Constraint: ").append(op.getPartitionConstraint()).append("\n");
+ List<IConnectorDescriptor> inputs = opInputMap.get(e.getKey());
+ if (inputs != null && !inputs.isEmpty()) {
+ buffer.append(" Inputs:\n");
+ for (IConnectorDescriptor c : inputs) {
+ buffer.append(" ").append(c.getConnectorId().getId()).append(" : ").append(c.toString())
+ .append("\n");
+ }
+ }
+ List<IConnectorDescriptor> outputs = opOutputMap.get(e.getKey());
+ if (outputs != null && !outputs.isEmpty()) {
+ buffer.append(" Outputs:\n");
+ for (IConnectorDescriptor c : outputs) {
+ buffer.append(" ").append(c.getConnectorId().getId()).append(" : ").append(c.toString())
+ .append("\n");
+ }
+ }
+ }
+
+ return buffer.toString();
+ }
+
+ public JSONObject toJSON() throws JSONException {
+ JSONObject jjob = new JSONObject();
+
+ jjob.put("type", "job");
+
+ JSONArray jopArray = new JSONArray();
+ for (Map.Entry<OperatorDescriptorId, IOperatorDescriptor> e : opMap.entrySet()) {
+ jopArray.put(e.getValue().toJSON());
+ }
+ jjob.put("operators", jopArray);
+
+ JSONArray jcArray = new JSONArray();
+ for (Map.Entry<ConnectorDescriptorId, IConnectorDescriptor> e : connMap.entrySet()) {
+ jcArray.put(e.getValue().toJSON());
+ }
+ jjob.put("connectors", jcArray);
+
+ JSONArray jconnectionsArray = new JSONArray();
+ for (Map.Entry<ConnectorDescriptorId, Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>>> e : connectorOpMap
+ .entrySet()) {
+ JSONObject conn = new JSONObject();
+ conn.put("type", "connection");
+ conn.put("connector-id", e.getValue().toString());
+ conn.put("in-operator-id", e.getValue().first.first.getOperatorId().toString());
+ conn.put("in-operator-port", e.getValue().first.second.intValue());
+ conn.put("out-operator-id", e.getValue().second.first.getOperatorId().toString());
+ conn.put("out-operator-port", e.getValue().second.second.intValue());
+ jconnectionsArray.put(conn);
+ }
+ jjob.put("connections", jconnectionsArray);
+
+ return jjob;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/JobStage.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/JobStage.java
new file mode 100644
index 0000000..41187f1
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/JobStage.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.job;
+
+import java.io.Serializable;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.UUID;
+
+import edu.uci.ics.hyracks.api.dataflow.ActivityNodeId;
+
+public class JobStage implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final UUID id;
+
+ private final Set<ActivityNodeId> tasks;
+
+ private final Set<JobStage> dependencies;
+
+ private final Set<JobStage> dependents;
+
+ private boolean started;
+
+ public JobStage(Set<ActivityNodeId> tasks) {
+ this.id = UUID.randomUUID();
+ this.tasks = tasks;
+ dependencies = new HashSet<JobStage>();
+ dependents = new HashSet<JobStage>();
+ }
+
+ public UUID getId() {
+ return id;
+ }
+
+ public Set<ActivityNodeId> getTasks() {
+ return tasks;
+ }
+
+ public void addDependency(JobStage stage) {
+ dependencies.add(stage);
+ }
+
+ public void addDependent(JobStage stage) {
+ dependents.add(stage);
+ }
+
+ public Set<JobStage> getDependencies() {
+ return dependencies;
+ }
+
+ @Override
+ public int hashCode() {
+ return id == null ? 0 : id.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof JobStage)) {
+ return false;
+ }
+ return id == ((JobStage) o).id;
+ }
+
+ @Override
+ public String toString() {
+ return "SID:" + id + ": " + tasks;
+ }
+
+ public boolean isStarted() {
+ return started;
+ }
+
+ public void setStarted() {
+ started = true;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/JobStatus.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/JobStatus.java
new file mode 100644
index 0000000..977f41a
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/JobStatus.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.job;
+
+public enum JobStatus {
+ INITIALIZED,
+ RUNNING,
+ TERMINATED,
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/statistics/JobStatistics.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/statistics/JobStatistics.java
new file mode 100644
index 0000000..921b239
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/statistics/JobStatistics.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.job.statistics;
+
+import java.io.Serializable;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+public class JobStatistics implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private Date startTime;
+
+ private Date endTime;
+
+ private List<StageStatistics> stages;
+
+ public JobStatistics() {
+ stages = new ArrayList<StageStatistics>();
+ }
+
+ public void setStartTime(Date startTime) {
+ this.startTime = startTime;
+ }
+
+ public Date getStartTime() {
+ return startTime;
+ }
+
+ public void setEndTime(Date endTime) {
+ this.endTime = endTime;
+ }
+
+ public Date getEndTime() {
+ return endTime;
+ }
+
+ public void addStageStatistics(StageStatistics stageStatistics) {
+ stages.add(stageStatistics);
+ }
+
+ public List<StageStatistics> getStages() {
+ return stages;
+ }
+
+ @Override
+ public String toString() {
+ DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
+ StringBuilder buffer = new StringBuilder();
+
+ buffer.append("{\n");
+ indent(buffer, 1).append("startTime: '").append(df.format(startTime)).append("',\n");
+ indent(buffer, 1).append("endTime: '").append(df.format(endTime)).append("',\n");
+ indent(buffer, 1).append("stages: [\n");
+ boolean first = true;
+ for (StageStatistics ss : stages) {
+ if (!first) {
+ buffer.append(",\n");
+ }
+ first = false;
+ ss.toString(buffer, 2);
+ }
+ buffer.append("\n");
+ indent(buffer, 1).append("]\n");
+ buffer.append("}");
+
+ return buffer.toString();
+ }
+
+ static StringBuilder indent(StringBuilder buffer, int level) {
+ for (int i = 0; i < level; ++i) {
+ buffer.append(" ");
+ }
+ return buffer;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/statistics/StageStatistics.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/statistics/StageStatistics.java
new file mode 100644
index 0000000..6cc437b
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/statistics/StageStatistics.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.job.statistics;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+public class StageStatistics implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private UUID stageId;
+
+ private Map<String, StageletStatistics> stageletMap;
+
+ public StageStatistics() {
+ stageletMap = new HashMap<String, StageletStatistics>();
+ }
+
+ public void setStageId(UUID stageId) {
+ this.stageId = stageId;
+ }
+
+ public UUID getStageId() {
+ return stageId;
+ }
+
+ public void addStageletStatistics(StageletStatistics ss) {
+ stageletMap.put(ss.getNodeId(), ss);
+ }
+
+ public Map<String, StageletStatistics> getStageletStatistics() {
+ return stageletMap;
+ }
+
+ public void toString(StringBuilder buffer, int level) {
+ JobStatistics.indent(buffer, level).append("{\n");
+ JobStatistics.indent(buffer, level + 1).append("stageId: '").append(stageId).append("',\n");
+ JobStatistics.indent(buffer, level + 1).append("stagelets: {\n");
+ boolean first = true;
+ for (Map.Entry<String, StageletStatistics> e : stageletMap.entrySet()) {
+ if (!first) {
+ buffer.append(",\n");
+ }
+ first = false;
+ JobStatistics.indent(buffer, level + 2).append(e.getKey()).append(": ");
+ e.getValue().toString(buffer, level + 3);
+ }
+ buffer.append("\n");
+ JobStatistics.indent(buffer, level + 1).append("}\n");
+ JobStatistics.indent(buffer, level).append("}");
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/statistics/StageletStatistics.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/statistics/StageletStatistics.java
new file mode 100644
index 0000000..66fb871
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/job/statistics/StageletStatistics.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.job.statistics;
+
+import java.io.Serializable;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Map;
+import java.util.TreeMap;
+
+public class StageletStatistics implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private String nodeId;
+
+ private Date startTime;
+
+ private Date endTime;
+
+ private Map<String, String> statisticsMap;
+
+ public StageletStatistics() {
+ statisticsMap = Collections.synchronizedMap(new TreeMap<String, String>());
+ }
+
+ public void setNodeId(String nodeId) {
+ this.nodeId = nodeId;
+ }
+
+ public String getNodeId() {
+ return nodeId;
+ }
+
+ public void setStartTime(Date startTime) {
+ this.startTime = startTime;
+ }
+
+ public Date getStartTime() {
+ return startTime;
+ }
+
+ public void setEndTime(Date endTime) {
+ this.endTime = endTime;
+ }
+
+ public Date getEndTime() {
+ return endTime;
+ }
+
+ public Map<String, String> getStatisticsMap() {
+ return statisticsMap;
+ }
+
+ public void toString(StringBuilder buffer, int level) {
+ DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
+ buffer.append("{\n");
+ JobStatistics.indent(buffer, level + 1).append("nodeId: '").append(nodeId).append("',\n");
+ JobStatistics.indent(buffer, level + 1).append("startTime: '").append(df.format(startTime)).append("',\n");
+ JobStatistics.indent(buffer, level + 1).append("endTime: '").append(df.format(endTime)).append("',\n");
+ JobStatistics.indent(buffer, level + 1).append("statistics: {\n");
+ boolean first = true;
+ for (Map.Entry<String, String> e : statisticsMap.entrySet()) {
+ if (!first) {
+ buffer.append(",\n");
+ }
+ first = false;
+ JobStatistics.indent(buffer, level + 2).append(e.getKey()).append(": '").append(e.getValue()).append("'");
+ }
+ buffer.append("\n");
+ JobStatistics.indent(buffer, level + 1).append("}\n");
+ JobStatistics.indent(buffer, level).append("}");
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java
new file mode 100644
index 0000000..2364d74
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.util;
+
+import java.io.Serializable;
+
+public final class Pair<F, S> implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ public F first;
+ public S second;
+
+ public Pair(F first, S second) {
+ this.first = first;
+ this.second = second;
+ }
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/ConnectionEntry.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/ConnectionEntry.java
new file mode 100644
index 0000000..4d9c309
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/ConnectionEntry.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.SocketChannel;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.comm.FrameConstants;
+import edu.uci.ics.hyracks.api.comm.IConnectionEntry;
+import edu.uci.ics.hyracks.api.comm.IDataReceiveListener;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public class ConnectionEntry implements IConnectionEntry {
+ private static final Logger LOGGER = Logger.getLogger(ConnectionEntry.class.getName());
+
+ private SocketChannel socketChannel;
+
+ private final ByteBuffer readBuffer;
+
+ private final ByteBuffer writeBuffer;
+
+ private IDataReceiveListener recvListener;
+
+ private Object attachment;
+
+ private final SelectionKey key;
+
+ public ConnectionEntry(HyracksContext ctx, SocketChannel socketChannel, SelectionKey key) {
+ this.socketChannel = socketChannel;
+ readBuffer = ctx.getResourceManager().allocateFrame();
+ readBuffer.clear();
+ writeBuffer = ctx.getResourceManager().allocateFrame();
+ writeBuffer.clear();
+ this.key = key;
+ }
+
+ public SocketChannel getSocketChannel() {
+ return socketChannel;
+ }
+
+ public boolean dispatch(SelectionKey key) throws IOException {
+ if (key.isReadable()) {
+ if (LOGGER.isLoggable(Level.FINER)) {
+ LOGGER.finer("Before read: " + readBuffer.position() + " " + readBuffer.limit());
+ }
+ int bytesRead = socketChannel.read(readBuffer);
+ if (bytesRead < 0) {
+ recvListener.eos(this);
+ return true;
+ }
+ if (LOGGER.isLoggable(Level.FINER)) {
+ LOGGER.finer("After read: " + readBuffer.position() + " " + readBuffer.limit());
+ }
+ recvListener.dataReceived(this);
+ } else if (key.isWritable()) {
+ synchronized (this) {
+ writeBuffer.flip();
+ if (LOGGER.isLoggable(Level.FINER)) {
+ LOGGER.finer("Before write: " + writeBuffer.position() + " " + writeBuffer.limit());
+ }
+ int bytesWritten = socketChannel.write(writeBuffer);
+ if (bytesWritten < 0) {
+ return true;
+ }
+ if (LOGGER.isLoggable(Level.FINER)) {
+ LOGGER.finer("After write: " + writeBuffer.position() + " " + writeBuffer.limit());
+ }
+ if (writeBuffer.remaining() <= 0) {
+ int ops = key.interestOps();
+ key.interestOps(ops & ~SelectionKey.OP_WRITE);
+ }
+ writeBuffer.compact();
+ notifyAll();
+ }
+ } else {
+ LOGGER.warning("Spurious event triggered: " + key.readyOps());
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public ByteBuffer getReadBuffer() {
+ return readBuffer;
+ }
+
+ @Override
+ public synchronized void write(ByteBuffer buffer) {
+ while (buffer.remaining() > 0) {
+ while (writeBuffer.remaining() <= 0) {
+ try {
+ wait();
+ } catch (InterruptedException e) {
+ }
+ }
+ int oldLimit = buffer.limit();
+ buffer.limit(Math.min(oldLimit, writeBuffer.remaining()));
+ writeBuffer.put(buffer);
+ buffer.limit(oldLimit);
+ int ops = key.interestOps();
+ key.interestOps(ops | SelectionKey.OP_WRITE);
+ key.selector().wakeup();
+ }
+ }
+
+ @Override
+ public void setDataReceiveListener(IDataReceiveListener listener) {
+ this.recvListener = listener;
+ }
+
+ @Override
+ public void attach(Object attachment) {
+ this.attachment = attachment;
+ }
+
+ @Override
+ public Object getAttachment() {
+ return attachment;
+ }
+
+ @Override
+ public void close() throws IOException {
+ socketChannel.close();
+ }
+
+ @Override
+ public SelectionKey getSelectionKey() {
+ return key;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/ConnectionManager.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/ConnectionManager.java
new file mode 100644
index 0000000..bfca6ad
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/ConnectionManager.java
@@ -0,0 +1,351 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.nio.ByteBuffer;
+import java.nio.channels.AsynchronousCloseException;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.comm.FrameConstants;
+import edu.uci.ics.hyracks.api.comm.IConnectionEntry;
+import edu.uci.ics.hyracks.api.comm.IDataReceiveListener;
+import edu.uci.ics.hyracks.api.comm.IDataReceiveListenerFactory;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.comm.NetworkAddress;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.comm.io.FrameHelper;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public class ConnectionManager {
+ private static final Logger LOGGER = Logger.getLogger(ConnectionManager.class.getName());
+
+ private static final int INITIAL_MESSAGE_LEN = 20;
+
+ private NetworkAddress networkAddress;
+
+ private ServerSocketChannel serverSocketChannel;
+
+ private final HyracksContext ctx;
+
+ private final Map<UUID, IDataReceiveListenerFactory> pendingConnectionReceivers;
+
+ private final ConnectionListenerThread connectionListenerThread;
+
+ private final DataListenerThread dataListenerThread;
+
+ private final IDataReceiveListener initialDataReceiveListener;
+
+ private volatile boolean stopped;
+
+ private ByteBuffer emptyFrame;
+
+ public ConnectionManager(HyracksContext ctx, InetAddress address) throws IOException {
+ this.ctx = ctx;
+ serverSocketChannel = ServerSocketChannel.open();
+ ServerSocket serverSocket = serverSocketChannel.socket();
+ serverSocket.bind(new InetSocketAddress(address, 0));
+
+ networkAddress = new NetworkAddress(serverSocket.getInetAddress(), serverSocket.getLocalPort());
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Connection manager listening on " + serverSocket.getInetAddress() + ":"
+ + serverSocket.getLocalPort());
+ }
+
+ pendingConnectionReceivers = new HashMap<UUID, IDataReceiveListenerFactory>();
+ dataListenerThread = new DataListenerThread();
+ connectionListenerThread = new ConnectionListenerThread();
+ initialDataReceiveListener = new InitialDataReceiveListener();
+ emptyFrame = ctx.getResourceManager().allocateFrame();
+ emptyFrame.putInt(FrameHelper.getTupleCountOffset(ctx), 0);
+ }
+
+ public synchronized void dumpStats() {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Number of pendingConnectionReceivers: " + pendingConnectionReceivers.size());
+ LOGGER.info("Number of selectable keys: " + dataListenerThread.selector.keys().size());
+ }
+ }
+
+ public NetworkAddress getNetworkAddress() {
+ return networkAddress;
+ }
+
+ public void start() {
+ stopped = false;
+ connectionListenerThread.start();
+ dataListenerThread.start();
+ }
+
+ public void stop() {
+ try {
+ stopped = true;
+ serverSocketChannel.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public IFrameWriter connect(NetworkAddress address, UUID id, int senderId) throws HyracksDataException {
+ try {
+ SocketChannel channel = SocketChannel
+ .open(new InetSocketAddress(address.getIpAddress(), address.getPort()));
+ byte[] initialFrame = new byte[INITIAL_MESSAGE_LEN];
+ ByteBuffer buffer = ByteBuffer.wrap(initialFrame);
+ buffer.clear();
+ buffer.putLong(id.getMostSignificantBits());
+ buffer.putLong(id.getLeastSignificantBits());
+ buffer.putInt(senderId);
+ buffer.flip();
+ int bytesWritten = 0;
+ while (bytesWritten < INITIAL_MESSAGE_LEN) {
+ int n = channel.write(buffer);
+ if (n < 0) {
+ throw new HyracksDataException("Stream closed prematurely");
+ }
+ bytesWritten += n;
+ }
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Send Initial message: " + id + ":" + senderId);
+ }
+ buffer.clear();
+ buffer.limit(FrameConstants.SIZE_LEN);
+ int bytesRead = 0;
+ while (bytesRead < FrameConstants.SIZE_LEN) {
+ int n = channel.read(buffer);
+ if (n < 0) {
+ throw new HyracksDataException("Stream closed prematurely");
+ }
+ bytesRead += n;
+ }
+ buffer.flip();
+ int frameLen = buffer.getInt();
+ if (frameLen != FrameConstants.SIZE_LEN) {
+ throw new IllegalStateException("Received illegal framelen = " + frameLen);
+ }
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Got Ack message: " + id + ":" + senderId);
+ }
+ return new NetworkFrameWriter(channel);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ public synchronized void acceptConnection(UUID id, IDataReceiveListenerFactory receiver) {
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.info("Connection manager accepting " + id);
+ }
+ pendingConnectionReceivers.put(id, receiver);
+ }
+
+ public synchronized void unacceptConnection(UUID id) {
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.info("Connection manager unaccepting " + id);
+ }
+ pendingConnectionReceivers.remove(id);
+ }
+
+ private final class NetworkFrameWriter implements IFrameWriter {
+ private SocketChannel channel;
+
+ NetworkFrameWriter(SocketChannel channel) {
+ this.channel = channel;
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ try {
+ synchronized (emptyFrame) {
+ emptyFrame.position(0);
+ emptyFrame.limit(emptyFrame.capacity());
+ channel.write(emptyFrame);
+ }
+ channel.close();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ try {
+ if (LOGGER.isLoggable(Level.FINER)) {
+ int frameLen = buffer.getInt(buffer.position());
+ LOGGER.finer("ConnectionManager.NetworkFrameWriter: frameLen = " + frameLen);
+ }
+ while (buffer.remaining() > 0) {
+ channel.write(buffer);
+ }
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ }
+ }
+
+ private final class ConnectionListenerThread extends Thread {
+ public ConnectionListenerThread() {
+ super("Hyracks Connection Listener Thread");
+ }
+
+ @Override
+ public void run() {
+ while (!stopped) {
+ try {
+ SocketChannel sc = serverSocketChannel.accept();
+ dataListenerThread.addSocketChannel(sc);
+ } catch (AsynchronousCloseException e) {
+ // do nothing
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ private final class DataListenerThread extends Thread {
+ private Selector selector;
+
+ private List<SocketChannel> pendingSockets;
+
+ public DataListenerThread() {
+ super("Hyracks Data Listener Thread");
+ try {
+ selector = Selector.open();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ pendingSockets = new ArrayList<SocketChannel>();
+ }
+
+ synchronized void addSocketChannel(SocketChannel sc) throws IOException {
+ LOGGER.info("Connection received");
+ pendingSockets.add(sc);
+ selector.wakeup();
+ }
+
+ @Override
+ public void run() {
+ while (!stopped) {
+ try {
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Starting Select");
+ }
+ int n = selector.select();
+ synchronized (this) {
+ if (!pendingSockets.isEmpty()) {
+ for (SocketChannel sc : pendingSockets) {
+ sc.configureBlocking(false);
+ SelectionKey scKey = sc.register(selector, SelectionKey.OP_READ);
+ ConnectionEntry entry = new ConnectionEntry(ctx, sc, scKey);
+ entry.setDataReceiveListener(initialDataReceiveListener);
+ scKey.attach(entry);
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Woke up selector");
+ }
+ }
+ pendingSockets.clear();
+ }
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Selector: " + n);
+ }
+ if (n > 0) {
+ for (Iterator<SelectionKey> i = selector.selectedKeys().iterator(); i.hasNext();) {
+ SelectionKey key = i.next();
+ i.remove();
+ ConnectionEntry entry = (ConnectionEntry) key.attachment();
+ boolean close = false;
+ try {
+ close = entry.dispatch(key);
+ } catch (IOException e) {
+ e.printStackTrace();
+ close = true;
+ }
+ if (close) {
+ key.cancel();
+ entry.close();
+ }
+ }
+ }
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ private class InitialDataReceiveListener implements IDataReceiveListener {
+ @Override
+ public void dataReceived(IConnectionEntry entry) throws IOException {
+ ByteBuffer buffer = entry.getReadBuffer();
+ buffer.flip();
+ IDataReceiveListener newListener = null;
+ if (buffer.remaining() >= INITIAL_MESSAGE_LEN) {
+ long msb = buffer.getLong();
+ long lsb = buffer.getLong();
+ UUID endpointID = new UUID(msb, lsb);
+ int senderId = buffer.getInt();
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Initial Frame received: " + endpointID + ":" + senderId);
+ }
+ IDataReceiveListenerFactory connectionReceiver;
+ synchronized (ConnectionManager.this) {
+ connectionReceiver = pendingConnectionReceivers.get(endpointID);
+ if (connectionReceiver == null) {
+ entry.close();
+ return;
+ }
+ }
+
+ newListener = connectionReceiver.getDataReceiveListener(endpointID, entry, senderId);
+ entry.setDataReceiveListener(newListener);
+ byte[] ack = new byte[4];
+ ByteBuffer ackBuffer = ByteBuffer.wrap(ack);
+ ackBuffer.clear();
+ ackBuffer.putInt(FrameConstants.SIZE_LEN);
+ ackBuffer.flip();
+ entry.write(ackBuffer);
+ }
+ buffer.compact();
+ if (newListener != null && buffer.remaining() > 0) {
+ newListener.dataReceived(entry);
+ }
+ }
+
+ @Override
+ public void eos(IConnectionEntry entry) {
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/DemuxDataReceiveListenerFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/DemuxDataReceiveListenerFactory.java
new file mode 100644
index 0000000..3eb2280
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/DemuxDataReceiveListenerFactory.java
@@ -0,0 +1,142 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.UUID;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.comm.IConnectionDemultiplexer;
+import edu.uci.ics.hyracks.api.comm.IConnectionEntry;
+import edu.uci.ics.hyracks.api.comm.IDataReceiveListener;
+import edu.uci.ics.hyracks.api.comm.IDataReceiveListenerFactory;
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public class DemuxDataReceiveListenerFactory implements IDataReceiveListenerFactory, IConnectionDemultiplexer,
+ IDataReceiveListener {
+ private static final Logger LOGGER = Logger.getLogger(DemuxDataReceiveListenerFactory.class.getName());
+
+ private final NonDeterministicFrameReader frameReader;
+ private final HyracksContext ctx;
+ private final BitSet readyBits;
+ private IConnectionEntry senders[];
+ private int openSenderCount;
+
+ public DemuxDataReceiveListenerFactory(HyracksContext ctx) {
+ frameReader = new NonDeterministicFrameReader(ctx, this);
+ this.ctx = ctx;
+ readyBits = new BitSet();
+ senders = null;
+ openSenderCount = 0;
+ }
+
+ @Override
+ public IDataReceiveListener getDataReceiveListener(UUID endpointUUID, IConnectionEntry entry, int senderIndex) {
+ entry.attach(senderIndex);
+ addSender(senderIndex, entry);
+ return this;
+ }
+
+ public IFrameReader getFrameReader() {
+ return frameReader;
+ }
+
+ @Override
+ public synchronized void dataReceived(IConnectionEntry entry) throws IOException {
+ int senderIndex = (Integer) entry.getAttachment();
+ ByteBuffer buffer = entry.getReadBuffer();
+ buffer.flip();
+ int dataLen = buffer.remaining();
+ if (dataLen >= ctx.getFrameSize()) {
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest("NonDeterministicDataReceiveListener: frame received: sender = " + senderIndex);
+ }
+ SelectionKey key = entry.getSelectionKey();
+ int ops = key.interestOps();
+ key.interestOps(ops & ~SelectionKey.OP_READ);
+ readyBits.set(senderIndex);
+ notifyAll();
+ return;
+ }
+ buffer.compact();
+ }
+
+ @Override
+ public void eos(IConnectionEntry entry) {
+ }
+
+ private synchronized void addSender(int senderIndex, IConnectionEntry entry) {
+ readyBits.clear(senderIndex);
+ if (senders == null) {
+ senders = new IConnectionEntry[senderIndex + 1];
+ } else if (senders.length <= senderIndex) {
+ senders = Arrays.copyOf(senders, senderIndex + 1);
+ }
+ senders[senderIndex] = entry;
+ ++openSenderCount;
+ }
+
+ @Override
+ public synchronized IConnectionEntry findNextReadyEntry(int lastReadSender) {
+ while (openSenderCount > 0 && readyBits.isEmpty()) {
+ try {
+ wait();
+ } catch (InterruptedException e) {
+ }
+ }
+ lastReadSender = readyBits.nextSetBit(lastReadSender);
+ if (lastReadSender < 0) {
+ lastReadSender = readyBits.nextSetBit(0);
+ }
+ return senders[lastReadSender];
+ }
+
+ @Override
+ public synchronized void unreadyEntry(int index) {
+ readyBits.clear(index);
+ IConnectionEntry entry = senders[index];
+ SelectionKey key = entry.getSelectionKey();
+ if (key.isValid()) {
+ int ops = key.interestOps();
+ key.interestOps(ops | SelectionKey.OP_READ);
+ key.selector().wakeup();
+ }
+ }
+
+ @Override
+ public synchronized int closeEntry(int index) throws HyracksDataException {
+ IConnectionEntry entry = senders[index];
+ SelectionKey key = entry.getSelectionKey();
+ key.cancel();
+ try {
+ entry.close();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ return --openSenderCount;
+ }
+
+ @Override
+ public synchronized int getSenderCount() {
+ return senders.length;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/IReceiverProtocolStack.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/IReceiverProtocolStack.java
new file mode 100644
index 0000000..b7effd5
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/IReceiverProtocolStack.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm;
+
+import edu.uci.ics.hyracks.api.comm.IDataReceiveListenerFactory;
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+
+public interface IReceiverProtocolStack {
+ public IDataReceiveListenerFactory createDataReceiveListenerFactory();
+
+ public IFrameReader createFrameReader();
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/ISenderProtocolStack.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/ISenderProtocolStack.java
new file mode 100644
index 0000000..c94c955
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/ISenderProtocolStack.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm;
+
+import edu.uci.ics.hyracks.api.comm.Endpoint;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface ISenderProtocolStack {
+ public IFrameWriter createFrameWriter(Endpoint endpoint) throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/NonDeterministicFrameReader.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/NonDeterministicFrameReader.java
new file mode 100644
index 0000000..3f2d449
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/NonDeterministicFrameReader.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm;
+
+import java.nio.ByteBuffer;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.comm.IConnectionDemultiplexer;
+import edu.uci.ics.hyracks.api.comm.IConnectionEntry;
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.comm.io.FrameHelper;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public class NonDeterministicFrameReader implements IFrameReader {
+ private static final Logger LOGGER = Logger.getLogger(NonDeterministicFrameReader.class.getName());
+
+ private final HyracksContext ctx;
+ private final IConnectionDemultiplexer demux;
+ private int lastReadSender;
+ private boolean eos;
+
+ public NonDeterministicFrameReader(HyracksContext ctx, IConnectionDemultiplexer demux) {
+ this.ctx = ctx;
+ this.demux = demux;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ lastReadSender = 0;
+ eos = false;
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ }
+
+ @Override
+ public boolean nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ if (eos) {
+ return false;
+ }
+ while (true) {
+ IConnectionEntry entry = demux.findNextReadyEntry(lastReadSender);
+ lastReadSender = (Integer) entry.getAttachment();
+ ByteBuffer netBuffer = entry.getReadBuffer();
+ int tupleCount = netBuffer.getInt(FrameHelper.getTupleCountOffset(ctx));
+ if (LOGGER.isLoggable(Level.FINER)) {
+ LOGGER.finer("Frame Tuple Count: " + tupleCount);
+ }
+ if (tupleCount == 0) {
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Empty Frame received: Closing " + lastReadSender);
+ }
+ int openEntries = demux.closeEntry(lastReadSender);
+ if (openEntries == 0) {
+ eos = true;
+ return false;
+ }
+ netBuffer.clear();
+ demux.unreadyEntry(lastReadSender);
+ } else {
+ buffer.clear();
+ buffer.put(netBuffer);
+ netBuffer.clear();
+ demux.unreadyEntry(lastReadSender);
+ return true;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/SortMergeFrameReader.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/SortMergeFrameReader.java
new file mode 100644
index 0000000..62372dc
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/SortMergeFrameReader.java
@@ -0,0 +1,247 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.Comparator;
+import java.util.PriorityQueue;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.comm.IConnectionDemultiplexer;
+import edu.uci.ics.hyracks.api.comm.IConnectionEntry;
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.comm.io.FrameHelper;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.comm.io.FrameTuplePairComparator;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public class SortMergeFrameReader implements IFrameReader {
+ private static final Logger LOGGER = Logger.getLogger(SortMergeFrameReader.class.getName());
+
+ private final HyracksContext ctx;
+ private final IConnectionDemultiplexer demux;
+ private final FrameTuplePairComparator tpc;
+ private final FrameTupleAppender appender;
+ private final RecordDescriptor recordDescriptor;
+ private Run[] runs;
+ private ByteBuffer[] frames;
+ private PriorityQueue<Integer> pQueue;
+ private int lastReadSender;
+ private boolean first;
+
+ public SortMergeFrameReader(HyracksContext ctx, IConnectionDemultiplexer demux, int[] sortFields,
+ IBinaryComparator[] comparators, RecordDescriptor recordDescriptor) {
+ this.ctx = ctx;
+ this.demux = demux;
+ tpc = new FrameTuplePairComparator(sortFields, sortFields, comparators);
+ appender = new FrameTupleAppender(ctx);
+ this.recordDescriptor = recordDescriptor;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ int nSenders = demux.getSenderCount();
+ runs = new Run[nSenders];
+ frames = new ByteBuffer[nSenders];
+ for (int i = 0; i < runs.length; ++i) {
+ runs[i] = new Run(i);
+ frames[i] = ctx.getResourceManager().allocateFrame();
+ }
+ pQueue = new PriorityQueue<Integer>(nSenders, new Comparator<Integer>() {
+ @Override
+ public int compare(Integer o1, Integer o2) {
+ int i1 = o1.intValue();
+ int i2 = o2.intValue();
+ Run r1 = runs[i1];
+ Run r2 = runs[i2];
+
+ int c = tpc.compare(r1.accessor, r1.tIndex, r2.accessor, r2.tIndex);
+ return c == 0 ? (i1 < i2 ? -1 : 1) : c;
+ }
+ });
+ lastReadSender = 0;
+ first = true;
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ for (Run r : runs) {
+ r.close();
+ }
+ }
+
+ @Override
+ public boolean nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ buffer.clear();
+ buffer.position(buffer.capacity());
+ appender.reset(buffer, true);
+ if (first) {
+ for (int i = 0; i < runs.length; ++i) {
+ if (runs[i].next()) {
+ pQueue.add(Integer.valueOf(i));
+ }
+ }
+ }
+ first = false;
+ while (true) {
+ if (pQueue.isEmpty()) {
+ return appender.getTupleCount() > 0;
+ }
+ Integer top = pQueue.peek();
+ Run run = runs[top.intValue()];
+ if (!appender.append(run.accessor, run.tIndex)) {
+ return true;
+ }
+ pQueue.remove();
+ if (run.next()) {
+ pQueue.add(top);
+ }
+ }
+ }
+
+ private class Run {
+ private final int runId;
+ private final File file;
+ private final FileChannel channel;
+ private final ByteBuffer frame;
+ private final FrameTupleAccessor accessor;
+ private int tIndex;
+ private long readFP;
+ private long writeFP;
+ private boolean eof;
+
+ public Run(int runId) throws HyracksDataException {
+ this.runId = runId;
+ try {
+ file = ctx.getResourceManager().createFile(SortMergeFrameReader.class.getSimpleName(), ".run");
+ RandomAccessFile raf = new RandomAccessFile(file, "rw");
+ channel = raf.getChannel();
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ frame = ctx.getResourceManager().allocateFrame();
+ accessor = new FrameTupleAccessor(ctx, recordDescriptor);
+ readFP = 0;
+ writeFP = 0;
+ eof = false;
+ }
+
+ public void close() throws HyracksDataException {
+ try {
+ channel.close();
+ file.delete();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ private void write(ByteBuffer frame) throws HyracksDataException {
+ try {
+ int len = frame.capacity();
+ while (len > 0) {
+ int sz = channel.write(frame, writeFP);
+ if (sz < 0) {
+ throw new HyracksDataException("Error writing to run");
+ }
+ len -= sz;
+ writeFP += sz;
+ }
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ private boolean next() throws HyracksDataException {
+ ++tIndex;
+ while (readFP == 0 || tIndex >= accessor.getTupleCount()) {
+ if (!read(frame)) {
+ return false;
+ }
+ accessor.reset(frame);
+ tIndex = 0;
+ }
+ return true;
+ }
+
+ private boolean read(ByteBuffer frame) throws HyracksDataException {
+ while (!eof && readFP >= writeFP) {
+ spoolRuns(runId);
+ }
+ if (eof && readFP >= writeFP) {
+ return false;
+ }
+ try {
+ channel.position(readFP);
+ frame.clear();
+ int len = frame.capacity();
+ while (len > 0) {
+ int sz = channel.read(frame, readFP);
+ if (sz < 0) {
+ throw new HyracksDataException("Error reading file");
+ }
+ len -= sz;
+ readFP += sz;
+ }
+ return true;
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ private void eof() {
+ eof = true;
+ }
+ }
+
+ private void spoolRuns(int interestingRun) throws HyracksDataException {
+ while (true) {
+ IConnectionEntry entry = demux.findNextReadyEntry(lastReadSender);
+ lastReadSender = (Integer) entry.getAttachment();
+ ByteBuffer netBuffer = entry.getReadBuffer();
+ int tupleCount = netBuffer.getInt(FrameHelper.getTupleCountOffset(ctx));
+ if (LOGGER.isLoggable(Level.FINER)) {
+ LOGGER.finer("Frame Tuple Count: " + tupleCount);
+ }
+ if (tupleCount == 0) {
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Empty Frame received: Closing " + lastReadSender);
+ }
+ int openEntries = demux.closeEntry(lastReadSender);
+ runs[lastReadSender].eof();
+ netBuffer.clear();
+ demux.unreadyEntry(lastReadSender);
+ if (openEntries == 0) {
+ return;
+ }
+ } else {
+ runs[lastReadSender].write(netBuffer);
+ netBuffer.clear();
+ demux.unreadyEntry(lastReadSender);
+ }
+ if (lastReadSender == interestingRun) {
+ return;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/ArrayTupleBuilder.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/ArrayTupleBuilder.java
new file mode 100644
index 0000000..d800fba
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/ArrayTupleBuilder.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm.io;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.api.comm.FrameConstants;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public class ArrayTupleBuilder {
+ private final ByteArrayAccessibleOutputStream baaos;
+ private final DataOutputStream dos;
+ private final int[] fEndOffsets;
+ private int nextField;
+
+ public ArrayTupleBuilder(int nFields) {
+ baaos = new ByteArrayAccessibleOutputStream();
+ dos = new DataOutputStream(baaos);
+ fEndOffsets = new int[nFields];
+ }
+
+ public void reset() {
+ nextField = 0;
+ baaos.reset();
+ }
+
+ public int[] getFieldEndOffsets() {
+ return fEndOffsets;
+ }
+
+ public byte[] getByteArray() {
+ return baaos.getByteArray();
+ }
+
+ public int getSize() {
+ return baaos.size();
+ }
+
+ public void addField(FrameTupleAccessor accessor, int tIndex, int fIndex) throws HyracksDataException {
+ int startOffset = accessor.getTupleStartOffset(tIndex);
+ int fStartOffset = accessor.getFieldStartOffset(tIndex, fIndex);
+ int fLen = accessor.getFieldEndOffset(tIndex, fIndex) - fStartOffset;
+ try {
+ dos.write(accessor.getBuffer().array(), startOffset + accessor.getFieldSlotsLength() + fStartOffset, fLen);
+ if (FrameConstants.DEBUG_FRAME_IO) {
+ dos.writeInt(FrameConstants.FRAME_FIELD_MAGIC);
+ }
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ fEndOffsets[nextField++] = baaos.size();
+ }
+
+ public <T> void addField(ISerializerDeserializer<T> serDeser, T instance) throws HyracksDataException {
+ serDeser.serialize(instance, dos);
+ fEndOffsets[nextField++] = baaos.size();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/ByteArrayAccessibleOutputStream.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/ByteArrayAccessibleOutputStream.java
new file mode 100644
index 0000000..41c771b
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/ByteArrayAccessibleOutputStream.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm.io;
+
+import java.io.ByteArrayOutputStream;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class ByteArrayAccessibleOutputStream extends ByteArrayOutputStream {
+ private static final Logger LOGGER = Logger.getLogger(ByteArrayAccessibleOutputStream.class.getName());
+
+ public byte[] getByteArray() {
+ return buf;
+ }
+
+ public void write(int b) {
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest("write(byte) value: " + b);
+ }
+ super.write(b);
+ }
+
+ @Override
+ public void write(byte[] bytes, int offset, int length) {
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest("write(byte[], int, int) offset: " + offset + " length" + length);
+ }
+ super.write(bytes, offset, length);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameDeserializer.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameDeserializer.java
new file mode 100644
index 0000000..22d4961
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameDeserializer.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm.io;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.comm.FrameConstants;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.comm.util.ByteBufferInputStream;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public class FrameDeserializer {
+ private static final Logger LOGGER = Logger.getLogger(FrameDeserializer.class.getName());
+
+ private final ByteBufferInputStream bbis;
+
+ private final DataInputStream di;
+
+ private final RecordDescriptor recordDescriptor;
+
+ private final FrameTupleAccessor frameTupleAccessor;
+
+ private int tupleCount;
+
+ private int tIndex;
+
+ private ByteBuffer buffer;
+
+ public FrameDeserializer(HyracksContext ctx, RecordDescriptor recordDescriptor) {
+ this.bbis = new ByteBufferInputStream();
+ this.di = new DataInputStream(bbis);
+ this.recordDescriptor = recordDescriptor;
+ frameTupleAccessor = new FrameTupleAccessor(ctx, recordDescriptor);
+ }
+
+ public void reset(ByteBuffer buffer) {
+ this.buffer = buffer;
+ frameTupleAccessor.reset(buffer);
+ tupleCount = frameTupleAccessor.getTupleCount();
+ tIndex = 0;
+ }
+
+ public boolean done() {
+ return tIndex >= tupleCount;
+ }
+
+ public Object[] deserializeRecord() throws HyracksDataException {
+ int start = frameTupleAccessor.getTupleStartOffset(tIndex) + frameTupleAccessor.getFieldSlotsLength();
+ bbis.setByteBuffer(buffer, start);
+ Object[] record = new Object[recordDescriptor.getFields().length];
+ for (int i = 0; i < record.length; ++i) {
+ Object instance = recordDescriptor.getFields()[i].deserialize(di);
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest(i + " " + instance);
+ }
+ record[i] = instance;
+ if (FrameConstants.DEBUG_FRAME_IO) {
+ try {
+ if (di.readInt() != FrameConstants.FRAME_FIELD_MAGIC) {
+ throw new HyracksDataException("Field magic mismatch");
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest("Read Record tIndex = " + tIndex + ", tupleCount = " + tupleCount);
+ }
+ ++tIndex;
+ return record;
+ }
+
+ public void close() {
+ try {
+ di.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameDeserializingDataReader.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameDeserializingDataReader.java
new file mode 100644
index 0000000..31d37b7
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameDeserializingDataReader.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm.io;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataReader;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public class FrameDeserializingDataReader implements IOpenableDataReader<Object[]> {
+ private final ByteBuffer buffer;
+
+ private boolean eos;
+
+ private boolean first;
+
+ private final IFrameReader frameReader;
+
+ private final FrameDeserializer frameDeserializer;
+
+ public FrameDeserializingDataReader(HyracksContext ctx, IFrameReader frameReader, RecordDescriptor recordDescriptor) {
+ buffer = ctx.getResourceManager().allocateFrame();
+ this.frameReader = frameReader;
+ this.frameDeserializer = new FrameDeserializer(ctx, recordDescriptor);
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ frameReader.open();
+ buffer.clear();
+ buffer.flip();
+ eos = false;
+ first = true;
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ frameReader.close();
+ frameDeserializer.close();
+ }
+
+ @Override
+ public Object[] readData() throws HyracksDataException {
+ while (true) {
+ if (eos) {
+ return null;
+ }
+ if (!first && !frameDeserializer.done()) {
+ return frameDeserializer.deserializeRecord();
+ }
+ buffer.clear();
+ if (!frameReader.nextFrame(buffer)) {
+ eos = true;
+ } else {
+ frameDeserializer.reset(buffer);
+ }
+ first = false;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameDeserializingDataWriter.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameDeserializingDataWriter.java
new file mode 100644
index 0000000..e0a7250
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameDeserializingDataWriter.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm.io;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataWriter;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public class FrameDeserializingDataWriter implements IFrameWriter {
+ private final IOpenableDataWriter<Object[]> writer;
+ private final FrameDeserializer frameDeserializer;
+
+ public FrameDeserializingDataWriter(HyracksContext ctx, IOpenableDataWriter<Object[]> writer,
+ RecordDescriptor recordDescriptor) {
+ this.writer = writer;
+ this.frameDeserializer = new FrameDeserializer(ctx, recordDescriptor);
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ writer.close();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ frameDeserializer.reset(buffer);
+ while (!frameDeserializer.done()) {
+ Object[] tuple = frameDeserializer.deserializeRecord();
+ writer.writeData(tuple);
+ }
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ writer.open();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameHelper.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameHelper.java
new file mode 100644
index 0000000..a82a051
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameHelper.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm.io;
+
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public class FrameHelper {
+ public static int getTupleCountOffset(HyracksContext ctx) {
+ return ctx.getFrameSize() - 4;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameTupleAccessor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameTupleAccessor.java
new file mode 100644
index 0000000..05d83cd
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameTupleAccessor.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm.io;
+
+import java.io.DataInputStream;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.comm.util.ByteBufferInputStream;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+/**
+ * FrameTupleCursor is used to navigate over tuples in a Frame.
+ * A frame is formatted with tuple data concatenated starting at offset 0, one tuple after another.
+ * Offset FS - 4 holds an int indicating the number of tuples (N) in the frame. FS - ((i + 1) * 4) for i from
+ * 0 to N - 1 holds an int indicating the offset of the (i + 1)^th tuple.
+ * Every tuple is organized as a sequence of shorts indicating the end of each field in the tuple relative to the end of the
+ * field slots.
+ *
+ * @author vinayakb
+ */
+public final class FrameTupleAccessor {
+ private final HyracksContext ctx;
+ private final RecordDescriptor recordDescriptor;
+
+ private ByteBuffer buffer;
+
+ public FrameTupleAccessor(HyracksContext ctx, RecordDescriptor recordDescriptor) {
+ this.ctx = ctx;
+ this.recordDescriptor = recordDescriptor;
+ }
+
+ public void reset(ByteBuffer buffer) {
+ this.buffer = buffer;
+ }
+
+ public ByteBuffer getBuffer() {
+ return buffer;
+ }
+
+ public int getTupleCount() {
+ return buffer.getInt(FrameHelper.getTupleCountOffset(ctx));
+ }
+
+ public int getTupleStartOffset(int tupleIndex) {
+ return tupleIndex == 0 ? 0 : buffer.getInt(FrameHelper.getTupleCountOffset(ctx) - 4 * tupleIndex);
+ }
+
+ public int getTupleEndOffset(int tupleIndex) {
+ return buffer.getInt(FrameHelper.getTupleCountOffset(ctx) - 4 * (tupleIndex + 1));
+ }
+
+ public int getFieldStartOffset(int tupleIndex, int fIdx) {
+ return fIdx == 0 ? 0 : buffer.getShort(getTupleStartOffset(tupleIndex) + (fIdx - 1) * 2);
+ }
+
+ public int getFieldEndOffset(int tupleIndex, int fIdx) {
+ return buffer.getShort(getTupleStartOffset(tupleIndex) + fIdx * 2);
+ }
+
+ public int getFieldSlotsLength() {
+ return recordDescriptor.getFields().length * 2;
+ }
+
+ public void prettyPrint() {
+ ByteBufferInputStream bbis = new ByteBufferInputStream();
+ DataInputStream dis = new DataInputStream(bbis);
+ int tc = getTupleCount();
+ System.err.println("TC: " + tc);
+ for (int i = 0; i < tc; ++i) {
+ System.err.print(i + ":(" + getTupleStartOffset(i) + ", " + getTupleEndOffset(i) + ")[");
+ for (int j = 0; j < recordDescriptor.getFields().length; ++j) {
+ System.err.print(j + ":(" + getFieldStartOffset(i, j) + ", " + getFieldEndOffset(i, j) + ") ");
+ System.err.print("{");
+ bbis.setByteBuffer(buffer, getTupleStartOffset(i) + getFieldSlotsLength() + getFieldStartOffset(i, j));
+ try {
+ System.err.print(recordDescriptor.getFields()[j].deserialize(dis));
+ } catch (HyracksDataException e) {
+ e.printStackTrace();
+ }
+ System.err.print("}");
+ }
+ System.err.println("]");
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameTupleAppender.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameTupleAppender.java
new file mode 100644
index 0000000..2ae094e
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameTupleAppender.java
@@ -0,0 +1,155 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm.io;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public class FrameTupleAppender {
+ private final HyracksContext ctx;
+
+ private ByteBuffer buffer;
+
+ private int tupleCount;
+
+ private int tupleDataEndOffset;
+
+ public FrameTupleAppender(HyracksContext ctx) {
+ this.ctx = ctx;
+ }
+
+ public void reset(ByteBuffer buffer, boolean clear) {
+ this.buffer = buffer;
+ if (clear) {
+ buffer.putInt(FrameHelper.getTupleCountOffset(ctx), 0);
+ tupleCount = 0;
+ tupleDataEndOffset = 0;
+ } else {
+ tupleCount = buffer.getInt(FrameHelper.getTupleCountOffset(ctx));
+ tupleDataEndOffset = tupleCount == 0 ? 0 : buffer.getInt(FrameHelper.getTupleCountOffset(ctx) - tupleCount
+ * 4);
+ }
+ }
+
+ public boolean append(int[] fieldSlots, byte[] bytes, int offset, int length) {
+ if (tupleDataEndOffset + fieldSlots.length * 2 + length + 4 + (tupleCount + 1) * 4 <= ctx.getFrameSize()) {
+ for (int i = 0; i < fieldSlots.length; ++i) {
+ buffer.putShort(tupleDataEndOffset + i * 2, (short) fieldSlots[i]);
+ }
+ System.arraycopy(bytes, offset, buffer.array(), tupleDataEndOffset + fieldSlots.length * 2, length);
+ tupleDataEndOffset += fieldSlots.length * 2 + length;
+ buffer.putInt(FrameHelper.getTupleCountOffset(ctx) - 4 * (tupleCount + 1), tupleDataEndOffset);
+ ++tupleCount;
+ buffer.putInt(FrameHelper.getTupleCountOffset(ctx), tupleCount);
+ return true;
+ }
+ return false;
+ }
+
+ public boolean append(FrameTupleAccessor tupleAccessor, int tIndex) {
+ int startOffset = tupleAccessor.getTupleStartOffset(tIndex);
+ int endOffset = tupleAccessor.getTupleEndOffset(tIndex);
+ int length = endOffset - startOffset;
+ if (tupleDataEndOffset + length + 4 + (tupleCount + 1) * 4 <= ctx.getFrameSize()) {
+ ByteBuffer src = tupleAccessor.getBuffer();
+ System.arraycopy(src.array(), startOffset, buffer.array(), tupleDataEndOffset, length);
+ tupleDataEndOffset += length;
+ buffer.putInt(FrameHelper.getTupleCountOffset(ctx) - 4 * (tupleCount + 1), tupleDataEndOffset);
+ ++tupleCount;
+ buffer.putInt(FrameHelper.getTupleCountOffset(ctx), tupleCount);
+ return true;
+ }
+ return false;
+ }
+
+ public boolean appendConcat(FrameTupleAccessor accessor0, int tIndex0, FrameTupleAccessor accessor1, int tIndex1) {
+ int startOffset0 = accessor0.getTupleStartOffset(tIndex0);
+ int endOffset0 = accessor0.getTupleEndOffset(tIndex0);
+ int length0 = endOffset0 - startOffset0;
+
+ int startOffset1 = accessor1.getTupleStartOffset(tIndex1);
+ int endOffset1 = accessor1.getTupleEndOffset(tIndex1);
+ int length1 = endOffset1 - startOffset1;
+
+ if (tupleDataEndOffset + length0 + length1 + 4 + (tupleCount + 1) * 4 <= ctx.getFrameSize()) {
+ ByteBuffer src0 = accessor0.getBuffer();
+ ByteBuffer src1 = accessor1.getBuffer();
+ int slotsLen0 = accessor0.getFieldSlotsLength();
+ int slotsLen1 = accessor1.getFieldSlotsLength();
+ int dataLen0 = length0 - slotsLen0;
+ int dataLen1 = length1 - slotsLen1;
+ // Copy slots from accessor0 verbatim
+ System.arraycopy(src0.array(), startOffset0, buffer.array(), tupleDataEndOffset, slotsLen0);
+ // Copy slots from accessor1 with the following transformation: newSlotIdx = oldSlotIdx + dataLen0
+ for (int i = 0; i < slotsLen1 / 2; ++i) {
+ buffer.putShort(tupleDataEndOffset + slotsLen0 + i * 2,
+ (short) (src1.getShort(startOffset1 + i * 2) + dataLen0));
+ }
+ // Copy data0
+ System.arraycopy(src0.array(), startOffset0 + slotsLen0, buffer.array(), tupleDataEndOffset + slotsLen0
+ + slotsLen1, dataLen0);
+ // Copy data1
+ System.arraycopy(src1.array(), startOffset1 + slotsLen1, buffer.array(), tupleDataEndOffset + slotsLen0
+ + slotsLen1 + dataLen0, dataLen1);
+ tupleDataEndOffset += (length0 + length1);
+ buffer.putInt(FrameHelper.getTupleCountOffset(ctx) - 4 * (tupleCount + 1), tupleDataEndOffset);
+ ++tupleCount;
+ buffer.putInt(FrameHelper.getTupleCountOffset(ctx), tupleCount);
+ return true;
+ }
+ return false;
+ }
+
+ public boolean appendProjection(FrameTupleAccessor accessor, int tIndex, int[] fields) {
+ int fTargetSlotsLength = fields.length * 2;
+ int length = fTargetSlotsLength;
+ for (int i = 0; i < fields.length; ++i) {
+ length += (accessor.getFieldEndOffset(tIndex, fields[i]) - accessor.getFieldStartOffset(tIndex, fields[i]));
+ }
+
+ if (tupleDataEndOffset + length + 4 + (tupleCount + 1) * 4 <= ctx.getFrameSize()) {
+ int fSrcSlotsLength = accessor.getFieldSlotsLength();
+ int tStartOffset = accessor.getTupleStartOffset(tIndex);
+
+ int fStartOffset = 0;
+ int fEndOffset = 0;
+ for (int i = 0; i < fields.length; ++i) {
+ int fSrcStart = tStartOffset + fSrcSlotsLength + accessor.getFieldStartOffset(tIndex, fields[i]);
+ int fLen = accessor.getFieldEndOffset(tIndex, fields[i])
+ - accessor.getFieldStartOffset(tIndex, fields[i]);
+ System.arraycopy(accessor.getBuffer().array(), fSrcStart, buffer.array(), tupleDataEndOffset
+ + fSrcSlotsLength + fStartOffset, fLen);
+ fEndOffset += fLen;
+ buffer.putShort(tupleDataEndOffset + i * 2, (short) fEndOffset);
+ fStartOffset = fEndOffset;
+ }
+ tupleDataEndOffset += length;
+ buffer.putInt(FrameHelper.getTupleCountOffset(ctx) - 4 * (tupleCount + 1), tupleDataEndOffset);
+ ++tupleCount;
+ buffer.putInt(FrameHelper.getTupleCountOffset(ctx), tupleCount);
+ return true;
+ }
+ return false;
+ }
+
+ public int getTupleCount() {
+ return tupleCount;
+ }
+
+ public ByteBuffer getBuffer() {
+ return buffer;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameTuplePairComparator.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameTuplePairComparator.java
new file mode 100644
index 0000000..0b23383
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/FrameTuplePairComparator.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm.io;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+
+public class FrameTuplePairComparator {
+ private final int[] keys0;
+ private final int[] keys1;
+ private final IBinaryComparator[] comparators;
+
+ public FrameTuplePairComparator(int[] keys0, int[] keys1, IBinaryComparator[] comparators) {
+ this.keys0 = keys0;
+ this.keys1 = keys1;
+ this.comparators = comparators;
+ }
+
+ public int compare(FrameTupleAccessor accessor0, int tIndex0, FrameTupleAccessor accessor1, int tIndex1) {
+ int tStart0 = accessor0.getTupleStartOffset(tIndex0);
+ int fStartOffset0 = accessor0.getFieldSlotsLength() + tStart0;
+
+ int tStart1 = accessor1.getTupleStartOffset(tIndex1);
+ int fStartOffset1 = accessor1.getFieldSlotsLength() + tStart1;
+
+ for (int i = 0; i < keys0.length; ++i) {
+ int fIdx0 = keys0[i];
+ int fStart0 = accessor0.getFieldStartOffset(tIndex0, fIdx0);
+ int fEnd0 = accessor0.getFieldEndOffset(tIndex0, fIdx0);
+ int fLen0 = fEnd0 - fStart0;
+
+ int fIdx1 = keys1[i];
+ int fStart1 = accessor1.getFieldStartOffset(tIndex1, fIdx1);
+ int fEnd1 = accessor1.getFieldEndOffset(tIndex1, fIdx1);
+ int fLen1 = fEnd1 - fStart1;
+
+ int c = comparators[i].compare(accessor0.getBuffer().array(), fStart0 + fStartOffset0, fLen0, accessor1
+ .getBuffer().array(), fStart1 + fStartOffset1, fLen1);
+ if (c != 0) {
+ return c;
+ }
+ }
+ return 0;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/SerializingDataWriter.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/SerializingDataWriter.java
new file mode 100644
index 0000000..4103dd6
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/io/SerializingDataWriter.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm.io;
+
+import java.nio.ByteBuffer;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataWriter;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public class SerializingDataWriter implements IOpenableDataWriter<Object[]> {
+ private static final Logger LOGGER = Logger.getLogger(SerializingDataWriter.class.getName());
+
+ private final ByteBuffer buffer;
+
+ private final ArrayTupleBuilder tb;
+
+ private final RecordDescriptor recordDescriptor;
+
+ private final IFrameWriter frameWriter;
+
+ private final FrameTupleAppender tupleAppender;
+
+ private boolean open;
+
+ public SerializingDataWriter(HyracksContext ctx, RecordDescriptor recordDescriptor, IFrameWriter frameWriter) {
+ buffer = ctx.getResourceManager().allocateFrame();
+ tb = new ArrayTupleBuilder(recordDescriptor.getFields().length);
+ this.recordDescriptor = recordDescriptor;
+ this.frameWriter = frameWriter;
+ tupleAppender = new FrameTupleAppender(ctx);
+ open = false;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ frameWriter.open();
+ buffer.clear();
+ open = true;
+ tupleAppender.reset(buffer, true);
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ if (!open) {
+ throw new HyracksDataException("Closing SerializingDataWriter that has not been opened");
+ }
+ if (tupleAppender.getTupleCount() > 0) {
+ flushFrame();
+ }
+ frameWriter.close();
+ open = false;
+ }
+
+ @Override
+ public void writeData(Object[] data) throws HyracksDataException {
+ if (!open) {
+ throw new HyracksDataException("Writing to SerializingDataWriter that has not been opened");
+ }
+ tb.reset();
+ for (int i = 0; i < data.length; ++i) {
+ Object instance = data[i];
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest(i + " " + instance);
+ }
+ tb.addField(recordDescriptor.getFields()[i], instance);
+ }
+ if (!tupleAppender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest("Flushing: position = " + buffer.position());
+ }
+ flushFrame();
+ tupleAppender.reset(buffer, true);
+ if (!tupleAppender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+
+ private void flushFrame() throws HyracksDataException {
+ buffer.position(0);
+ buffer.limit(buffer.capacity());
+ frameWriter.nextFrame(buffer);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/util/ByteBufferInputStream.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/util/ByteBufferInputStream.java
new file mode 100644
index 0000000..90f7433
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/util/ByteBufferInputStream.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm.util;
+
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class ByteBufferInputStream extends InputStream {
+ private static final Logger LOGGER = Logger.getLogger(ByteBufferInputStream.class.getName());
+
+ private ByteBuffer buffer;
+
+ private int position;
+
+ public ByteBufferInputStream() {
+ }
+
+ public void setByteBuffer(ByteBuffer buffer, int position) {
+ this.buffer = buffer;
+ this.position = position;
+ }
+
+ @Override
+ public int read() {
+ int remaining = buffer.capacity() - position;
+ int value = remaining > 0 ? (buffer.get(position++) & 0xff) : -1;
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest("read(): value: " + value + " remaining: " + remaining + " position: " + position);
+ }
+ return value;
+ }
+
+ @Override
+ public int read(byte[] bytes, int offset, int length) {
+ int remaining = buffer.capacity() - position;
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest("read(bytes[], int, int): remaining: " + remaining + " offset: " + offset + " length: "
+ + length + " position: " + position);
+ }
+ if (remaining == 0) {
+ return -1;
+ }
+ int l = Math.min(length, remaining);
+ System.arraycopy(buffer.array(), position, bytes, offset, l);
+ position += l;
+ return l;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/util/FrameUtils.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/util/FrameUtils.java
new file mode 100644
index 0000000..bdce4a4
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/comm/util/FrameUtils.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.comm.util;
+
+import java.nio.ByteBuffer;
+
+public class FrameUtils {
+ public static void copy(ByteBuffer srcFrame, ByteBuffer destFrame) {
+ makeReadable(srcFrame);
+ destFrame.clear();
+ destFrame.put(srcFrame);
+ }
+
+ public static void makeReadable(ByteBuffer frame) {
+ frame.position(0);
+ frame.limit(frame.capacity());
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/config/CCConfig.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/config/CCConfig.java
new file mode 100644
index 0000000..950569d
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/config/CCConfig.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.config;
+
+import org.kohsuke.args4j.Option;
+
+public class CCConfig {
+ @Option(name = "-port", usage = "Sets the port to listen for connections from node controllers (default 1099)")
+ public int port = 1099;
+
+ @Option(name = "-http-port", usage = "Sets the http port for the admin console")
+ public int httpPort;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/config/NCConfig.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/config/NCConfig.java
new file mode 100644
index 0000000..c95f1ad
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/config/NCConfig.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.config;
+
+import java.io.Serializable;
+
+import org.kohsuke.args4j.Option;
+
+public class NCConfig implements Serializable{
+ @Option(name = "-cc-host", usage = "Cluster Controller host name")
+ public String ccHost;
+
+ @Option(name = "-cc-port", usage = "Cluster Controller port (default: 1099)")
+ public int ccPort = 1099;
+
+ @Option(name = "-node-id", usage = "Logical name of node controller unique within the cluster")
+ public String nodeId;
+
+ @Option(name = "-data-ip-address", usage = "IP Address to bind data listener")
+ public String dataIPAddress;
+
+ @Option(name = "-frame-size", usage = "Frame Size to use for data communication (default: 32768)")
+ public int frameSize = 32768;
+
+ @Option(name = "-dcache-client-servers", usage = "Sets the list of DCache servers in the format host1:port1,host2:port2,... (default localhost:54583)")
+ public String dcacheClientServers = "localhost:54583";
+
+ @Option(name = "-dcache-client-server-local", usage = "Sets the local DCache server, if one is available, in the format host:port (default not set)")
+ public String dcacheClientServerLocal;
+
+ @Option(name = "-dcache-client-path", usage = "Sets the path to store the files retrieved from the DCache server (default /tmp/dcache-client)")
+ public String dcacheClientPath = "/tmp/dcache-client";
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/context/HyracksContext.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/context/HyracksContext.java
new file mode 100644
index 0000000..7fe088d
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/context/HyracksContext.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.context;
+
+import edu.uci.ics.hyracks.resources.ResourceManager;
+
+public class HyracksContext {
+ private final ResourceManager resourceManager;
+ private final int frameSize;
+
+ public HyracksContext(int frameSize) {
+ resourceManager = new ResourceManager(this);
+ this.frameSize = frameSize;
+ }
+
+ public ResourceManager getResourceManager() {
+ return resourceManager;
+ }
+
+ public int getFrameSize() {
+ return frameSize;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/controller/AbstractRemoteService.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/controller/AbstractRemoteService.java
new file mode 100644
index 0000000..9c8f3dd
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/controller/AbstractRemoteService.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.controller;
+
+import java.rmi.RemoteException;
+import java.rmi.server.UnicastRemoteObject;
+
+import edu.uci.ics.hyracks.service.IService;
+
+public abstract class AbstractRemoteService extends UnicastRemoteObject implements IService {
+ private static final long serialVersionUID = 1L;
+
+ public AbstractRemoteService() throws RemoteException {
+ }
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/controller/ClusterControllerService.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/controller/ClusterControllerService.java
new file mode 100644
index 0000000..0461499
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/controller/ClusterControllerService.java
@@ -0,0 +1,208 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.controller;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.rmi.registry.LocateRegistry;
+import java.rmi.registry.Registry;
+import java.util.EnumSet;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.eclipse.jetty.server.Handler;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.handler.AbstractHandler;
+import org.eclipse.jetty.server.handler.ContextHandler;
+
+import edu.uci.ics.hyracks.api.controller.IClusterController;
+import edu.uci.ics.hyracks.api.controller.INodeController;
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.api.job.statistics.JobStatistics;
+import edu.uci.ics.hyracks.api.job.statistics.StageletStatistics;
+import edu.uci.ics.hyracks.config.CCConfig;
+import edu.uci.ics.hyracks.job.JobManager;
+import edu.uci.ics.hyracks.web.WebServer;
+
+public class ClusterControllerService extends AbstractRemoteService implements IClusterController {
+ private static final long serialVersionUID = 1L;
+
+ private CCConfig ccConfig;
+
+ private static Logger LOGGER = Logger.getLogger(ClusterControllerService.class.getName());
+
+ private final Map<String, INodeController> nodeRegistry;
+
+ private WebServer webServer;
+
+ private final JobManager jobManager;
+
+ private final Executor taskExecutor;
+
+ public ClusterControllerService(CCConfig ccConfig) throws Exception {
+ this.ccConfig = ccConfig;
+ nodeRegistry = new LinkedHashMap<String, INodeController>();
+ jobManager = new JobManager(this);
+ taskExecutor = Executors.newCachedThreadPool();
+ webServer = new WebServer(new Handler[] { getAdminConsoleHandler() });
+ }
+
+ @Override
+ public void start() throws Exception {
+ LOGGER.log(Level.INFO, "Starting ClusterControllerService");
+ Registry registry = LocateRegistry.createRegistry(ccConfig.port);
+ registry.rebind(IClusterController.class.getName(), this);
+ webServer.setPort(ccConfig.httpPort);
+ webServer.start();
+ LOGGER.log(Level.INFO, "Started ClusterControllerService");
+ }
+
+ @Override
+ public void stop() throws Exception {
+ LOGGER.log(Level.INFO, "Stopping ClusterControllerService");
+ webServer.stop();
+ LOGGER.log(Level.INFO, "Stopped ClusterControllerService");
+ }
+
+ @Override
+ public UUID createJob(JobSpecification jobSpec) throws Exception {
+ return jobManager.createJob(jobSpec, EnumSet.noneOf(JobFlag.class));
+ }
+
+ @Override
+ public UUID createJob(JobSpecification jobSpec, EnumSet<JobFlag> jobFlags) throws Exception {
+ return jobManager.createJob(jobSpec, jobFlags);
+ }
+
+ @Override
+ public void registerNode(INodeController nodeController) throws Exception {
+ String id = nodeController.getId();
+ synchronized (this) {
+ if (nodeRegistry.containsKey(id)) {
+ throw new Exception("Node with this name already registered.");
+ }
+ nodeRegistry.put(id, nodeController);
+ }
+ nodeController.notifyRegistration(this);
+ LOGGER.log(Level.INFO, "Registered INodeController: id = " + id);
+ }
+
+ @Override
+ public void unregisterNode(INodeController nodeController) throws Exception {
+ String id = nodeController.getId();
+ synchronized (this) {
+ nodeRegistry.remove(id);
+ }
+ LOGGER.log(Level.INFO, "Unregistered INodeController");
+ }
+
+ @Override
+ public INodeController lookupNode(String id) throws Exception {
+ return nodeRegistry.get(id);
+ }
+
+ public Executor getExecutor() {
+ return taskExecutor;
+ }
+
+ public synchronized void notifyJobComplete(final UUID jobId) {
+ for (final INodeController nc : nodeRegistry.values()) {
+ taskExecutor.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ nc.cleanUpJob(jobId);
+ } catch (Exception e) {
+ }
+ }
+
+ });
+ }
+ }
+
+ @Override
+ public void notifyStageletComplete(UUID jobId, UUID stageId, String nodeId, StageletStatistics statistics)
+ throws Exception {
+ jobManager.notifyStageletComplete(jobId, stageId, nodeId, statistics);
+ }
+
+ @Override
+ public JobStatus getJobStatus(UUID jobId) throws Exception {
+ return jobManager.getJobStatus(jobId);
+ }
+
+ @Override
+ public void start(UUID jobId) throws Exception {
+ jobManager.start(jobId);
+ }
+
+ @Override
+ public JobStatistics waitForCompletion(UUID jobId) throws Exception {
+ return jobManager.waitForCompletion(jobId);
+ }
+
+ private Handler getAdminConsoleHandler() {
+ ContextHandler handler = new ContextHandler("/admin");
+ handler.setHandler(new AbstractHandler() {
+ @Override
+ public void handle(String target, Request baseRequest, HttpServletRequest request,
+ HttpServletResponse response) throws IOException, ServletException {
+ if (!"/".equals(target)) {
+ return;
+ }
+ response.setContentType("text/html;charset=utf-8");
+ response.setStatus(HttpServletResponse.SC_OK);
+ baseRequest.setHandled(true);
+ PrintWriter writer = response.getWriter();
+ writer.println("<html><head><title>Hyracks Admin Console</title></head><body>");
+ writer.println("<h1>Hyracks Admin Console</h1>");
+ writer.println("<h2>Node Controllers</h2>");
+ writer.println("<table><tr><td>Node Id</td><td>Host</td></tr>");
+ synchronized (ClusterControllerService.this) {
+ for (Map.Entry<String, INodeController> e : nodeRegistry.entrySet()) {
+ try {
+ INodeController n = e.getValue();
+ writer.print("<tr><td>");
+ writer.print(e.getKey());
+ writer.print("</td><td>");
+ writer.print("</td></tr>");
+ } catch (Exception ex) {
+ }
+ }
+ }
+ writer.println("</table>");
+ writer.println("</body></html>");
+ writer.flush();
+ }
+ });
+ return handler;
+ }
+
+ @Override
+ public Map<String, INodeController> getRegistry() throws Exception {
+ return nodeRegistry;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/controller/NodeControllerService.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/controller/NodeControllerService.java
new file mode 100644
index 0000000..dbdf1c9
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/controller/NodeControllerService.java
@@ -0,0 +1,374 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.controller;
+
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.rmi.registry.LocateRegistry;
+import java.rmi.registry.Registry;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import edu.uci.ics.hyracks.api.comm.Endpoint;
+import edu.uci.ics.hyracks.api.comm.IConnectionDemultiplexer;
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.controller.IClusterController;
+import edu.uci.ics.hyracks.api.controller.INodeController;
+import edu.uci.ics.hyracks.api.controller.NodeCapability;
+import edu.uci.ics.hyracks.api.dataflow.ActivityNodeId;
+import edu.uci.ics.hyracks.api.dataflow.IActivityNode;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IEndpointDataWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.OperatorInstanceId;
+import edu.uci.ics.hyracks.api.dataflow.PortInstanceId;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.JobStage;
+import edu.uci.ics.hyracks.api.job.statistics.StageletStatistics;
+import edu.uci.ics.hyracks.comm.ConnectionManager;
+import edu.uci.ics.hyracks.comm.DemuxDataReceiveListenerFactory;
+import edu.uci.ics.hyracks.config.NCConfig;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.job.Joblet;
+import edu.uci.ics.hyracks.job.Stagelet;
+import edu.uci.ics.hyracks.runtime.OperatorRunnable;
+
+public class NodeControllerService extends AbstractRemoteService implements INodeController {
+ private static final long serialVersionUID = 1L;
+
+ private NCConfig ncConfig;
+
+ private final String id;
+
+ private final HyracksContext ctx;
+
+ private final NodeCapability nodeCapability;
+
+ private final ConnectionManager connectionManager;
+
+ private IClusterController ccs;
+
+ private Map<UUID, Joblet> jobletMap;
+
+ private Executor executor;
+
+ public NodeControllerService(NCConfig ncConfig) throws Exception {
+ this.ncConfig = ncConfig;
+ id = ncConfig.nodeId;
+ this.ctx = new HyracksContext(ncConfig.frameSize);
+ if (id == null) {
+ throw new Exception("id not set");
+ }
+ nodeCapability = computeNodeCapability();
+ connectionManager = new ConnectionManager(ctx, getIpAddress(ncConfig));
+ jobletMap = new HashMap<UUID, Joblet>();
+ executor = Executors.newCachedThreadPool();
+ }
+
+ private static Logger LOGGER = Logger.getLogger(NodeControllerService.class.getName());
+
+ @Override
+ public void start() throws Exception {
+ LOGGER.log(Level.INFO, "Starting NodeControllerService");
+ connectionManager.start();
+ Registry registry = LocateRegistry.getRegistry(ncConfig.ccHost, ncConfig.ccPort);
+ IClusterController cc = (IClusterController) registry.lookup(IClusterController.class.getName());
+ cc.registerNode(this);
+ LOGGER.log(Level.INFO, "Started NodeControllerService");
+ }
+
+ @Override
+ public void stop() throws Exception {
+ LOGGER.log(Level.INFO, "Stopping NodeControllerService");
+ connectionManager.stop();
+ LOGGER.log(Level.INFO, "Stopped NodeControllerService");
+ }
+
+ @Override
+ public String getId() throws Exception {
+ return id;
+ }
+
+ @Override
+ public NodeCapability getNodeCapability() throws Exception {
+ return nodeCapability;
+ }
+
+ public ConnectionManager getConnectionManager() {
+ return connectionManager;
+ }
+
+ private static NodeCapability computeNodeCapability() {
+ NodeCapability nc = new NodeCapability();
+ nc.setCPUCount(Runtime.getRuntime().availableProcessors());
+ return nc;
+ }
+
+ private static InetAddress getIpAddress(NCConfig ncConfig) throws Exception {
+ String ipaddrStr = ncConfig.dataIPAddress;
+ ipaddrStr = ipaddrStr.trim();
+ Pattern pattern = Pattern.compile("(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})");
+ Matcher m = pattern.matcher(ipaddrStr);
+ if (!m.matches()) {
+ throw new Exception(MessageFormat.format(
+ "Connection Manager IP Address String %s does is not a valid IP Address.", ipaddrStr));
+ }
+ byte[] ipBytes = new byte[4];
+ ipBytes[0] = (byte) Integer.parseInt(m.group(1));
+ ipBytes[1] = (byte) Integer.parseInt(m.group(2));
+ ipBytes[2] = (byte) Integer.parseInt(m.group(3));
+ ipBytes[3] = (byte) Integer.parseInt(m.group(4));
+ return InetAddress.getByAddress(ipBytes);
+ }
+
+ @Override
+ public Map<PortInstanceId, Endpoint> initializeJobletPhase1(UUID jobId, JobPlan plan, JobStage stage)
+ throws Exception {
+ LOGGER.log(Level.INFO, String.valueOf(jobId) + "[" + id + ":" + stage.getId()
+ + "]: Initializing Joblet Phase 1");
+
+ final Joblet joblet = getLocalJoblet(jobId);
+
+ Stagelet stagelet = new Stagelet(joblet, stage.getId(), id);
+ joblet.setStagelet(stage.getId(), stagelet);
+
+ final Map<PortInstanceId, Endpoint> portMap = new HashMap<PortInstanceId, Endpoint>();
+ Map<OperatorInstanceId, OperatorRunnable> honMap = stagelet.getOperatorMap();
+
+ List<Endpoint> endpointList = new ArrayList<Endpoint>();
+
+ for (ActivityNodeId hanId : stage.getTasks()) {
+ IActivityNode han = plan.getActivityNodeMap().get(hanId);
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest("Initializing " + hanId + " -> " + han);
+ }
+ IOperatorDescriptor op = han.getOwner();
+ List<IConnectorDescriptor> inputs = plan.getTaskInputs(hanId);
+ String[] partitions = op.getPartitions();
+ for (int i = 0; i < partitions.length; ++i) {
+ String part = partitions[i];
+ if (id.equals(part)) {
+ IOperatorNodePushable hon = han.createPushRuntime(ctx, plan, joblet.getEnvironment(op, i), i);
+ OperatorRunnable or = new OperatorRunnable(ctx, hon);
+ stagelet.setOperator(op.getOperatorId(), i, or);
+ if (inputs != null) {
+ for (int j = 0; j < inputs.size(); ++j) {
+ if (j >= 1) {
+ throw new IllegalStateException();
+ }
+ IConnectorDescriptor conn = inputs.get(j);
+ Endpoint endpoint = new Endpoint(connectionManager.getNetworkAddress(), i);
+ endpointList.add(endpoint);
+ DemuxDataReceiveListenerFactory drlf = new DemuxDataReceiveListenerFactory(ctx);
+ connectionManager.acceptConnection(endpoint.getEndpointId(), drlf);
+ PortInstanceId piId = new PortInstanceId(op.getOperatorId(),
+ PortInstanceId.Direction.INPUT, plan.getTaskInputMap().get(hanId).get(j), i);
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest("Created endpoint " + piId + " -> " + endpoint);
+ }
+ portMap.put(piId, endpoint);
+ IFrameReader reader = createReader(conn, drlf, i, plan, stagelet);
+ or.setFrameReader(reader);
+ }
+ }
+ honMap.put(new OperatorInstanceId(op.getOperatorId(), i), or);
+ }
+ }
+ }
+
+ stagelet.setEndpointList(endpointList);
+
+ return portMap;
+ }
+
+ private IFrameReader createReader(final IConnectorDescriptor conn, IConnectionDemultiplexer demux,
+ final int receiverIndex, JobPlan plan, final Stagelet stagelet) throws HyracksDataException {
+ final IFrameReader reader = conn.createReceiveSideReader(ctx, plan, demux, receiverIndex);
+
+ return plan.getJobFlags().contains(JobFlag.COLLECT_FRAME_COUNTS) ? new IFrameReader() {
+ private int frameCount;
+
+ @Override
+ public void open() throws HyracksDataException {
+ frameCount = 0;
+ reader.open();
+ }
+
+ @Override
+ public boolean nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ boolean status = reader.nextFrame(buffer);
+ if (status) {
+ ++frameCount;
+ }
+ return status;
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ reader.close();
+ stagelet.getStatistics().getStatisticsMap().put(
+ "framecount." + conn.getConnectorId().getId() + ".receiver." + receiverIndex,
+ String.valueOf(frameCount));
+ }
+ } : reader;
+ }
+
+ @Override
+ public void initializeJobletPhase2(UUID jobId, final JobPlan plan, JobStage stage,
+ final Map<PortInstanceId, Endpoint> globalPortMap) throws Exception {
+ LOGGER.log(Level.INFO, String.valueOf(jobId) + "[" + id + ":" + stage.getId()
+ + "]: Initializing Joblet Phase 2");
+ final Joblet ji = getLocalJoblet(jobId);
+ Stagelet si = (Stagelet) ji.getStagelet(stage.getId());
+ final Map<OperatorInstanceId, OperatorRunnable> honMap = si.getOperatorMap();
+
+ final Stagelet stagelet = (Stagelet) ji.getStagelet(stage.getId());
+
+ final JobSpecification spec = plan.getJobSpecification();
+
+ for (ActivityNodeId hanId : stage.getTasks()) {
+ IActivityNode han = plan.getActivityNodeMap().get(hanId);
+ IOperatorDescriptor op = han.getOwner();
+ List<IConnectorDescriptor> outputs = plan.getTaskOutputs(hanId);
+ String[] partitions = op.getPartitions();
+ for (int i = 0; i < partitions.length; ++i) {
+ String part = partitions[i];
+ if (id.equals(part)) {
+ OperatorRunnable or = honMap.get(new OperatorInstanceId(op.getOperatorId(), i));
+ if (outputs != null) {
+ for (int j = 0; j < outputs.size(); ++j) {
+ final IConnectorDescriptor conn = outputs.get(j);
+ final int senderIndex = i;
+ IEndpointDataWriterFactory edwFactory = new IEndpointDataWriterFactory() {
+ @Override
+ public IFrameWriter createFrameWriter(int index) throws HyracksDataException {
+ PortInstanceId piId = new PortInstanceId(spec.getConsumer(conn).getOperatorId(),
+ PortInstanceId.Direction.INPUT, spec.getConsumerInputIndex(conn), index);
+ Endpoint ep = globalPortMap.get(piId);
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest("Probed endpoint " + piId + " -> " + ep);
+ }
+ return createWriter(connectionManager.connect(ep.getNetworkAddress(), ep
+ .getEndpointId(), senderIndex), plan, conn, senderIndex, index, stagelet);
+ }
+ };
+ or.setFrameWriter(j, conn.createSendSideWriter(ctx, plan, edwFactory, i));
+ }
+ }
+ stagelet.installRunnable(new OperatorInstanceId(op.getOperatorId(), i));
+ }
+ }
+ }
+ }
+
+ private IFrameWriter createWriter(final IFrameWriter writer, JobPlan plan, final IConnectorDescriptor conn,
+ final int senderIndex, final int receiverIndex, final Stagelet stagelet) throws HyracksDataException {
+ return plan.getJobFlags().contains(JobFlag.COLLECT_FRAME_COUNTS) ? new IFrameWriter() {
+ private int frameCount;
+
+ @Override
+ public void open() throws HyracksDataException {
+ frameCount = 0;
+ writer.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ ++frameCount;
+ writer.nextFrame(buffer);
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ writer.close();
+ stagelet.getStatistics().getStatisticsMap().put(
+ "framecount." + conn.getConnectorId().getId() + ".sender." + senderIndex + "." + receiverIndex,
+ String.valueOf(frameCount));
+ }
+ } : writer;
+ }
+
+ @Override
+ public void commitJobletInitialization(UUID jobId, JobPlan plan, JobStage stage) throws Exception {
+ final Joblet ji = getLocalJoblet(jobId);
+ Stagelet si = (Stagelet) ji.getStagelet(stage.getId());
+ for (Endpoint e : si.getEndpointList()) {
+ connectionManager.unacceptConnection(e.getEndpointId());
+ }
+ si.setEndpointList(null);
+ }
+
+ private synchronized Joblet getLocalJoblet(UUID jobId) throws Exception {
+ Joblet ji = jobletMap.get(jobId);
+ if (ji == null) {
+ ji = new Joblet(this, jobId);
+ jobletMap.put(jobId, ji);
+ }
+ return ji;
+ }
+
+ public Executor getExecutor() {
+ return executor;
+ }
+
+ @Override
+ public synchronized void cleanUpJob(UUID jobId) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Cleaning up after job: " + jobId);
+ }
+ jobletMap.remove(jobId);
+ connectionManager.dumpStats();
+ }
+
+ @Override
+ public void startStage(UUID jobId, UUID stageId) throws Exception {
+ Joblet ji = jobletMap.get(jobId);
+ if (ji != null) {
+ Stagelet s = ji.getStagelet(stageId);
+ if (s != null) {
+ s.start();
+ }
+ }
+ }
+
+ public void notifyStageComplete(UUID jobId, UUID stageId, StageletStatistics stats) throws Exception {
+ ccs.notifyStageletComplete(jobId, stageId, id, stats);
+ }
+
+ @Override
+ public void notifyRegistration(IClusterController ccs) throws Exception {
+ this.ccs = ccs;
+ }
+
+ @Override
+ public NCConfig getConfiguration() throws Exception {
+ return ncConfig;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/ExternalSortOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/ExternalSortOperatorDescriptor.java
new file mode 100644
index 0000000..e5052b2
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/ExternalSortOperatorDescriptor.java
@@ -0,0 +1,643 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePullable;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.AbstractActivityNode;
+import edu.uci.ics.hyracks.coreops.base.AbstractOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.util.ReferenceEntry;
+import edu.uci.ics.hyracks.coreops.util.ReferencedPriorityQueue;
+
+public class ExternalSortOperatorDescriptor extends AbstractOperatorDescriptor {
+ private static final String IN_FRAMES = "inFrames";
+ private static final String TPOINTERS = "tPointers";
+ private static final String RUNS = "runs";
+
+ private static final long serialVersionUID = 1L;
+ private final int[] sortFields;
+ private IBinaryComparatorFactory[] comparatorFactories;
+ private final int framesLimit;
+
+ public ExternalSortOperatorDescriptor(JobSpecification spec, int framesLimit, int[] sortFields,
+ IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDescriptor) {
+ super(spec, 1, 1);
+ this.framesLimit = framesLimit;
+ this.sortFields = sortFields;
+ this.comparatorFactories = comparatorFactories;
+ if (framesLimit <= 1) {
+ throw new IllegalStateException();// minimum of 2 fames (1 in,1 out)
+ }
+ recordDescriptors[0] = recordDescriptor;
+ }
+
+ @Override
+ public void contributeTaskGraph(IActivityGraphBuilder builder) {
+ SortActivity sa = new SortActivity();
+ MergeActivity ma = new MergeActivity();
+
+ builder.addTask(sa);
+ builder.addSourceEdge(0, sa, 0);
+
+ builder.addTask(ma);
+ builder.addTargetEdge(0, ma, 0);
+
+ builder.addBlockingEdge(sa, ma);
+ }
+
+ private class SortActivity extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IOperatorDescriptor getOwner() {
+ return ExternalSortOperatorDescriptor.this;
+ }
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ return null;
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final HyracksContext ctx, JobPlan plan,
+ final IOperatorEnvironment env, int partition) {
+ final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
+ for (int i = 0; i < comparatorFactories.length; ++i) {
+ comparators[i] = comparatorFactories[i].createBinaryComparator();
+ }
+ IOperatorNodePushable op = new IOperatorNodePushable() {
+ private final FrameTupleAccessor fta1 = new FrameTupleAccessor(ctx, recordDescriptors[0]);
+ private final FrameTupleAccessor fta2 = new FrameTupleAccessor(ctx, recordDescriptors[0]);
+ private List<ByteBuffer> inFrames;
+ private ByteBuffer outFrame;
+ private LinkedList<File> runs;
+ private int activeInFrame;
+
+ @Override
+ public void open() throws HyracksDataException {
+ inFrames = new ArrayList<ByteBuffer>();
+ outFrame = ctx.getResourceManager().allocateFrame();
+ runs = new LinkedList<File>();
+ activeInFrame = 0;
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ if (activeInFrame + 1 >= framesLimit) { // + 1 outFrame.
+ try {
+ createRunFromInFrames(inFrames.size());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ ByteBuffer copy;
+ buffer.position(0);
+ buffer.limit(buffer.capacity());
+ if (runs.size() <= 0) {
+ copy = ctx.getResourceManager().allocateFrame();
+ copy.put(buffer);
+ inFrames.add(copy);
+ } else {
+ copy = inFrames.get(activeInFrame);
+ copy.put(buffer);
+ }
+ ++activeInFrame;
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ env.set(IN_FRAMES, inFrames);
+ env.set(RUNS, runs);
+ if (activeInFrame > 0) {
+ if (runs.size() <= 0) {
+ long[] tPointers = getSortedTPointers(activeInFrame);
+ env.set(TPOINTERS, tPointers);
+ } else {
+ createRunFromInFrames(activeInFrame);
+ }
+ }
+ }
+
+ private void createRunFromInFrames(int nBuffers) throws HyracksDataException {
+ File runFile;
+ try {
+ runFile = ctx.getResourceManager().createFile(
+ ExternalSortOperatorDescriptor.class.getSimpleName(), ".run");
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ RunFileWriter writer = new RunFileWriter(runFile);
+ writer.open();
+ try {
+ flushFrames(ctx, inFrames, outFrame, getSortedTPointers(nBuffers), writer);
+ } finally {
+ writer.close();
+ }
+ runs.add(runFile);
+ activeInFrame = 0;
+ }
+
+ private long[] getSortedTPointers(int nBuffers) {
+ FrameTupleAccessor accessor = new FrameTupleAccessor(ctx, recordDescriptors[0]);
+ int totalTCount = 0;
+ for (int i = 0; i < nBuffers; ++i) {
+ accessor.reset(inFrames.get(i));
+ totalTCount += accessor.getTupleCount();
+ }
+ long[] tPointers = new long[totalTCount];
+ int ptr = 0;
+ for (int i = 0; i < nBuffers; ++i) {
+ accessor.reset(inFrames.get(i));
+ int tCount = accessor.getTupleCount();
+ for (int j = 0; j < tCount; ++j) {
+ tPointers[ptr++] = (((long) i) << 32) + j;
+ }
+ }
+ if (tPointers.length > 0) {
+ sort(tPointers, 0, tPointers.length);
+ }
+ return tPointers;
+ }
+
+ private void sort(long[] tPointers, int offset, int length) {
+ int m = offset + (length >> 1);
+ long v = tPointers[m];
+
+ int a = offset;
+ int b = a;
+ int c = offset + length - 1;
+ int d = c;
+ while (true) {
+ while (b <= c && compare(tPointers[b], v) <= 0) {
+ if (compare(tPointers[b], v) == 0) {
+ swap(tPointers, a++, b);
+ }
+ ++b;
+ }
+ while (c >= b && compare(tPointers[c], v) >= 0) {
+ if (compare(tPointers[c], v) == 0) {
+ swap(tPointers, c, d--);
+ }
+ --c;
+ }
+ if (b > c)
+ break;
+ swap(tPointers, b++, c--);
+ }
+
+ int s;
+ int n = offset + length;
+ s = Math.min(a - offset, b - a);
+ vecswap(tPointers, offset, b - s, s);
+ s = Math.min(d - c, n - d - 1);
+ vecswap(tPointers, b, n - s, s);
+
+ if ((s = b - a) > 1) {
+ sort(tPointers, offset, s);
+ }
+ if ((s = d - c) > 1) {
+ sort(tPointers, n - s, s);
+ }
+ }
+
+ private void swap(long x[], int a, int b) {
+ long t = x[a];
+ x[a] = x[b];
+ x[b] = t;
+ }
+
+ private void vecswap(long x[], int a, int b, int n) {
+ for (int i = 0; i < n; i++, a++, b++) {
+ swap(x, a, b);
+ }
+ }
+
+ private int compare(long tp1, long tp2) {
+ int i1 = (int) ((tp1 >> 32) & 0xffffffff);
+ int j1 = (int) (tp1 & 0xffffffff);
+ int i2 = (int) ((tp2 >> 32) & 0xffffffff);
+ int j2 = (int) (tp2 & 0xffffffff);
+ ByteBuffer buf1 = inFrames.get(i1);
+ ByteBuffer buf2 = inFrames.get(i2);
+ byte[] b1 = buf1.array();
+ byte[] b2 = buf2.array();
+ fta1.reset(buf1);
+ fta2.reset(buf2);
+ for (int f = 0; f < sortFields.length; ++f) {
+ int fIdx = sortFields[f];
+ int s1 = fta1.getTupleStartOffset(j1) + fta1.getFieldSlotsLength()
+ + fta1.getFieldStartOffset(j1, fIdx);
+ int l1 = fta1.getFieldEndOffset(j1, fIdx) - fta1.getFieldStartOffset(j1, fIdx);
+ int s2 = fta2.getTupleStartOffset(j2) + fta2.getFieldSlotsLength()
+ + fta2.getFieldStartOffset(j2, fIdx);
+ int l2 = fta2.getFieldEndOffset(j2, fIdx) - fta2.getFieldStartOffset(j2, fIdx);
+ int c = comparators[f].compare(b1, s1, l1, b2, s2, l2);
+ if (c != 0) {
+ return c;
+ }
+ }
+ return 0;
+ }
+
+ @Override
+ public void setFrameWriter(int index, IFrameWriter writer) {
+ throw new IllegalArgumentException();
+ }
+ };
+ return op;
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return false;
+ }
+ }
+
+ private class MergeActivity extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IOperatorDescriptor getOwner() {
+ return ExternalSortOperatorDescriptor.this;
+ }
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ return null;
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final HyracksContext ctx, JobPlan plan,
+ final IOperatorEnvironment env, int partition) {
+ final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
+ for (int i = 0; i < comparatorFactories.length; ++i) {
+ comparators[i] = comparatorFactories[i].createBinaryComparator();
+ }
+ IOperatorNodePushable op = new IOperatorNodePushable() {
+ private IFrameWriter writer;
+ private List<ByteBuffer> inFrames;
+ private ByteBuffer outFrame;
+ LinkedList<File> runs;
+ private FrameTupleAppender outFrameAppender;
+
+ @Override
+ public void open() throws HyracksDataException {
+ inFrames = (List<ByteBuffer>) env.get(IN_FRAMES);
+ outFrame = ctx.getResourceManager().allocateFrame();
+ runs = (LinkedList<File>) env.get(RUNS);
+ outFrameAppender = new FrameTupleAppender(ctx);
+ outFrameAppender.reset(outFrame, true);
+ writer.open();
+ try {
+ if (runs.size() <= 0) {
+ long[] tPointers = (long[]) env.get(TPOINTERS);
+ if (tPointers != null) {
+ flushFrames(ctx, inFrames, outFrame, tPointers, writer);
+ env.set(TPOINTERS, null);
+ }
+ } else {
+ int passCount = 0;
+ while (runs.size() > 0) {
+ passCount++;
+ try {
+ doPass(runs, passCount);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ } finally {
+ writer.close();
+ }
+ env.set(IN_FRAMES, null);
+ env.set(RUNS, null);
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ // do nothing
+ }
+
+ @Override
+ public void setFrameWriter(int index, IFrameWriter writer) {
+ if (index != 0) {
+ throw new IllegalArgumentException();
+ }
+ this.writer = writer;
+ }
+
+ // creates a new run from runs that can fit in memory.
+ private void doPass(LinkedList<File> runs, int passCount) throws ClassNotFoundException, Exception {
+ File newRun = null;
+ IFrameWriter writer = this.writer;
+ boolean finalPass = false;
+ if (runs.size() + 1 <= framesLimit) { // + 1 outFrame
+ finalPass = true;
+ for (int i = inFrames.size() - 1; i >= runs.size(); i--) {
+ inFrames.remove(i);
+ }
+ } else {
+ newRun = ctx.getResourceManager().createFile(
+ ExternalSortOperatorDescriptor.class.getSimpleName(), ".run");
+ writer = new RunFileWriter(newRun);
+ }
+ RunFileReader[] runCursors = new RunFileReader[inFrames.size()];
+ FrameTupleAccessor[] tupleAccessors = new FrameTupleAccessor[inFrames.size()];
+ Comparator<ReferenceEntry> comparator = createEntryComparator(comparators);
+ ReferencedPriorityQueue topTuples = new ReferencedPriorityQueue(ctx, recordDescriptors[0],
+ inFrames.size(), comparator);
+ int[] tupleIndexes = new int[inFrames.size()];
+ for (int i = 0; i < inFrames.size(); i++) {
+ tupleIndexes[i] = 0;
+ int runIndex = topTuples.peek().getRunid();
+ runCursors[runIndex] = new RunFileReader(runs.get(runIndex));
+ runCursors[runIndex].open();
+ if (runCursors[runIndex].nextFrame(inFrames.get(runIndex))) {
+ tupleAccessors[runIndex] = new FrameTupleAccessor(ctx, recordDescriptors[0]);
+ tupleAccessors[runIndex].reset(inFrames.get(runIndex));
+ setNextTopTuple(runIndex, tupleIndexes, runCursors, tupleAccessors, topTuples);
+ } else {
+ closeRun(runIndex, runCursors, tupleAccessors);
+ }
+ }
+
+ while (!topTuples.areRunsExhausted()) {
+ ReferenceEntry top = topTuples.peek();
+ int runIndex = top.getRunid();
+ FrameTupleAccessor fta = top.getAccessor();
+ int tupleIndex = top.getTupleIndex();
+
+ if (!outFrameAppender.append(fta, tupleIndex)) {
+ flushFrame(outFrame, writer);
+ outFrameAppender.reset(outFrame, true);
+ if (!outFrameAppender.append(fta, tupleIndex)) {
+ throw new IllegalStateException();
+ }
+ }
+
+ ++tupleIndexes[runIndex];
+ setNextTopTuple(runIndex, tupleIndexes, runCursors, tupleAccessors, topTuples);
+ }
+ if (outFrameAppender.getTupleCount() > 0) {
+ flushFrame(outFrame, writer);
+ outFrameAppender.reset(outFrame, true);
+ }
+ runs.subList(0, inFrames.size()).clear();
+ if (!finalPass) {
+ runs.add(0, newRun);
+ }
+ }
+
+ private void setNextTopTuple(int runIndex, int[] tupleIndexes, RunFileReader[] runCursors,
+ FrameTupleAccessor[] tupleAccessors, ReferencedPriorityQueue topTuples) throws IOException {
+ boolean exists = hasNextTuple(runIndex, tupleIndexes, runCursors, tupleAccessors);
+ if (exists) {
+ topTuples.popAndReplace(tupleAccessors[runIndex], tupleIndexes[runIndex]);
+ } else {
+ topTuples.pop();
+ closeRun(runIndex, runCursors, tupleAccessors);
+ }
+ }
+
+ private boolean hasNextTuple(int runIndex, int[] tupleIndexes, RunFileReader[] runCursors,
+ FrameTupleAccessor[] tupleAccessors) throws IOException {
+ if (tupleAccessors[runIndex] == null || runCursors[runIndex] == null) {
+ return false;
+ } else if (tupleIndexes[runIndex] >= tupleAccessors[runIndex].getTupleCount()) {
+ ByteBuffer buf = tupleAccessors[runIndex].getBuffer(); // same-as-inFrames.get(runIndex)
+ if (runCursors[runIndex].nextFrame(buf)) {
+ tupleIndexes[runIndex] = 0;
+ return hasNextTuple(runIndex, tupleIndexes, runCursors, tupleAccessors);
+ } else {
+ return false;
+ }
+ } else {
+ return true;
+ }
+ }
+
+ private void closeRun(int index, RunFileReader[] runCursors, FrameTupleAccessor[] tupleAccessor) {
+ runCursors[index] = null;
+ tupleAccessor[index] = null;
+ }
+ };
+ return op;
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+ }
+
+ private Comparator<ReferenceEntry> createEntryComparator(final IBinaryComparator[] comparators) {
+ return new Comparator<ReferenceEntry>() {
+ public int compare(ReferenceEntry tp1, ReferenceEntry tp2) {
+ FrameTupleAccessor fta1 = (FrameTupleAccessor) tp1.getAccessor();
+ FrameTupleAccessor fta2 = (FrameTupleAccessor) tp2.getAccessor();
+ int j1 = (Integer) tp1.getTupleIndex();
+ int j2 = (Integer) tp2.getTupleIndex();
+ byte[] b1 = fta1.getBuffer().array();
+ byte[] b2 = fta2.getBuffer().array();
+ for (int f = 0; f < sortFields.length; ++f) {
+ int fIdx = sortFields[f];
+ int s1 = fta1.getTupleStartOffset(j1) + fta1.getFieldSlotsLength()
+ + fta1.getFieldStartOffset(j1, fIdx);
+ int l1 = fta1.getFieldEndOffset(j1, fIdx) - fta1.getFieldStartOffset(j1, fIdx);
+ int s2 = fta2.getTupleStartOffset(j2) + fta2.getFieldSlotsLength()
+ + fta2.getFieldStartOffset(j2, fIdx);
+ int l2 = fta2.getFieldEndOffset(j2, fIdx) - fta2.getFieldStartOffset(j2, fIdx);
+ int c = comparators[f].compare(b1, s1, l1, b2, s2, l2);
+ if (c != 0) {
+ return c;
+ }
+ }
+ return 0;
+ }
+ };
+ }
+
+ private void flushFrames(HyracksContext ctx, List<ByteBuffer> inFrames, ByteBuffer outFrame, long[] tPointers,
+ IFrameWriter writer) throws HyracksDataException {
+ FrameTupleAccessor accessor = new FrameTupleAccessor(ctx, recordDescriptors[0]);
+ FrameTupleAppender outFrameAppender = new FrameTupleAppender(ctx);
+ for (ByteBuffer buf : inFrames) {
+ buf.position(0);
+ buf.limit(buf.capacity());
+ }
+ outFrameAppender.reset(outFrame, true);
+ for (int ptr = 0; ptr < tPointers.length; ++ptr) {
+ long tp = tPointers[ptr];
+ int i = (int) ((tp >> 32) & 0xffffffff);
+ int j = (int) (tp & 0xffffffff);
+ ByteBuffer buffer = inFrames.get(i);
+ accessor.reset(buffer);
+ if (!outFrameAppender.append(accessor, j)) {
+ flushFrame(outFrame, writer);
+ outFrameAppender.reset(outFrame, true);
+ if (!outFrameAppender.append(accessor, j)) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+ if (outFrameAppender.getTupleCount() > 0) {
+ flushFrame(outFrame, writer);
+ outFrame.position(0);
+ outFrame.limit(outFrame.capacity());
+ }
+ }
+
+ private void flushFrame(ByteBuffer frame, IFrameWriter writer) throws HyracksDataException {
+ frame.position(0);
+ frame.limit(frame.capacity());
+ writer.nextFrame(frame);
+ }
+
+ private class RunFileWriter implements IFrameWriter {
+ private final File file;
+ private FileChannel channel;
+
+ public RunFileWriter(File file) {
+ this.file = file;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ RandomAccessFile raf;
+ try {
+ raf = new RandomAccessFile(file, "rw");
+ } catch (FileNotFoundException e) {
+ throw new HyracksDataException(e);
+ }
+ channel = raf.getChannel();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ int remain = buffer.capacity();
+ while (remain > 0) {
+ int len;
+ try {
+ len = channel.write(buffer);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ if (len < 0) {
+ throw new HyracksDataException("Error writing data");
+ }
+ remain -= len;
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ try {
+ channel.close();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+
+ public static class RunFileReader implements IFrameReader {
+ private final File file;
+ private FileChannel channel;
+
+ public RunFileReader(File file) throws FileNotFoundException {
+ this.file = file;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ RandomAccessFile raf;
+ try {
+ raf = new RandomAccessFile(file, "r");
+ } catch (FileNotFoundException e) {
+ throw new HyracksDataException(e);
+ }
+ channel = raf.getChannel();
+ }
+
+ @Override
+ public boolean nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ buffer.clear();
+ int remain = buffer.capacity();
+ while (remain > 0) {
+ int len;
+ try {
+ len = channel.read(buffer);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ if (len < 0) {
+ return false;
+ }
+ remain -= len;
+ }
+ return true;
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ try {
+ channel.close();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/FieldHashPartitionComputerFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/FieldHashPartitionComputerFactory.java
new file mode 100644
index 0000000..ab66fb4
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/FieldHashPartitionComputerFactory.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunction;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAccessor;
+
+public class FieldHashPartitionComputerFactory implements ITuplePartitionComputerFactory {
+ private static final long serialVersionUID = 1L;
+ private final int[] hashFields;
+ private final IBinaryHashFunctionFactory[] hashFunctionFactories;
+
+ public FieldHashPartitionComputerFactory(int[] hashFields, IBinaryHashFunctionFactory[] hashFunctionFactories) {
+ this.hashFields = hashFields;
+ this.hashFunctionFactories = hashFunctionFactories;
+ }
+
+ @Override
+ public ITuplePartitionComputer createPartitioner() {
+ final IBinaryHashFunction[] hashFunctions = new IBinaryHashFunction[hashFunctionFactories.length];
+ for (int i = 0; i < hashFunctionFactories.length; ++i) {
+ hashFunctions[i] = hashFunctionFactories[i].createBinaryHashFunction();
+ }
+ return new ITuplePartitionComputer() {
+ @Override
+ public int partition(FrameTupleAccessor accessor, int tIndex, int nParts) {
+ int h = 0;
+ int startOffset = accessor.getTupleStartOffset(tIndex);
+ int slotLength = accessor.getFieldSlotsLength();
+ for (int j = 0; j < hashFields.length; ++j) {
+ int fIdx = hashFields[j];
+ IBinaryHashFunction hashFn = hashFunctions[j];
+ int fStart = accessor.getFieldStartOffset(tIndex, fIdx);
+ int fEnd = accessor.getFieldEndOffset(tIndex, fIdx);
+ int fh = hashFn
+ .hash(accessor.getBuffer().array(), startOffset + slotLength + fStart, fEnd - fStart);
+ h += fh;
+ }
+ if (h < 0) {
+ h = -h;
+ }
+ return h % nParts;
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/HashDataWriter.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/HashDataWriter.java
new file mode 100644
index 0000000..2c1e28a
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/HashDataWriter.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.IEndpointDataWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public class HashDataWriter implements IFrameWriter {
+ private final int consumerPartitionCount;
+ private final IFrameWriter[] epWriters;
+ private final FrameTupleAppender[] appenders;
+ private final FrameTupleAccessor tupleAccessor;
+ private final ITuplePartitionComputer tpc;
+
+ public HashDataWriter(HyracksContext ctx, int consumerPartitionCount, IEndpointDataWriterFactory edwFactory,
+ RecordDescriptor recordDescriptor, ITuplePartitionComputer tpc) throws HyracksDataException {
+ this.consumerPartitionCount = consumerPartitionCount;
+ epWriters = new IFrameWriter[consumerPartitionCount];
+ appenders = new FrameTupleAppender[consumerPartitionCount];
+ for (int i = 0; i < consumerPartitionCount; ++i) {
+ try {
+ epWriters[i] = edwFactory.createFrameWriter(i);
+ appenders[i] = new FrameTupleAppender(ctx);
+ appenders[i].reset(ctx.getResourceManager().allocateFrame(), true);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ tupleAccessor = new FrameTupleAccessor(ctx, recordDescriptor);
+ this.tpc = tpc;
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ for (int i = 0; i < epWriters.length; ++i) {
+ if (appenders[i].getTupleCount() > 0) {
+ flushFrame(appenders[i].getBuffer(), epWriters[i]);
+ }
+ epWriters[i].close();
+ }
+ }
+
+ private void flushFrame(ByteBuffer buffer, IFrameWriter frameWriter) throws HyracksDataException {
+ buffer.position(0);
+ buffer.limit(buffer.capacity());
+ frameWriter.nextFrame(buffer);
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ for (int i = 0; i < epWriters.length; ++i) {
+ epWriters[i].open();
+ appenders[i].reset(appenders[i].getBuffer(), true);
+ }
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ tupleAccessor.reset(buffer);
+ int tupleCount = tupleAccessor.getTupleCount();
+ for (int i = 0; i < tupleCount; ++i) {
+ int h = tpc.partition(tupleAccessor, i, consumerPartitionCount);
+ FrameTupleAppender appender = appenders[h];
+ if (!appender.append(tupleAccessor, i)) {
+ ByteBuffer appenderBuffer = appender.getBuffer();
+ flushFrame(appenderBuffer, epWriters[h]);
+ appender.reset(appenderBuffer, true);
+ if (!appender.append(tupleAccessor, i)) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/IMapper.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/IMapper.java
new file mode 100644
index 0000000..aec6135
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/IMapper.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import edu.uci.ics.hyracks.api.dataflow.IDataWriter;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface IMapper {
+ public void map(Object[] data, IDataWriter<Object[]> writer) throws HyracksDataException;
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/IMapperFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/IMapperFactory.java
new file mode 100644
index 0000000..d73a3fc
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/IMapperFactory.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface IMapperFactory extends Serializable {
+ public IMapper createMapper() throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/ITupleAggregator.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/ITupleAggregator.java
new file mode 100644
index 0000000..c482591
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/ITupleAggregator.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import edu.uci.ics.hyracks.api.dataflow.IDataWriter;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface ITupleAggregator {
+ void add(Object[] data);
+
+ void init(Object[] data);
+
+ void write(IDataWriter<Object[]> writer) throws HyracksDataException;
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/ITupleAggregatorFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/ITupleAggregatorFactory.java
new file mode 100644
index 0000000..161232b
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/ITupleAggregatorFactory.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface ITupleAggregatorFactory extends Serializable {
+ public ITupleAggregator createTupleAggregator() throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/InMemorySortOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/InMemorySortOperatorDescriptor.java
new file mode 100644
index 0000000..45e882a
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/InMemorySortOperatorDescriptor.java
@@ -0,0 +1,321 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePullable;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.comm.util.FrameUtils;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.AbstractActivityNode;
+import edu.uci.ics.hyracks.coreops.base.AbstractOperatorDescriptor;
+
+public class InMemorySortOperatorDescriptor extends AbstractOperatorDescriptor {
+ private static final String BUFFERS = "buffers";
+ private static final String TPOINTERS = "tpointers";
+
+ private static final long serialVersionUID = 1L;
+ private final int[] sortFields;
+ private IBinaryComparatorFactory[] comparatorFactories;
+
+ public InMemorySortOperatorDescriptor(JobSpecification spec, int[] sortFields,
+ IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDescriptor) {
+ super(spec, 1, 1);
+ this.sortFields = sortFields;
+ this.comparatorFactories = comparatorFactories;
+ recordDescriptors[0] = recordDescriptor;
+ }
+
+ @Override
+ public void contributeTaskGraph(IActivityGraphBuilder builder) {
+ SortActivity sa = new SortActivity();
+ MergeActivity ma = new MergeActivity();
+
+ builder.addTask(sa);
+ builder.addSourceEdge(0, sa, 0);
+
+ builder.addTask(ma);
+ builder.addTargetEdge(0, ma, 0);
+
+ builder.addBlockingEdge(sa, ma);
+ }
+
+ private class SortActivity extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IOperatorDescriptor getOwner() {
+ return InMemorySortOperatorDescriptor.this;
+ }
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ return null;
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final HyracksContext ctx, JobPlan plan,
+ final IOperatorEnvironment env, int partition) {
+ final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
+ for (int i = 0; i < comparatorFactories.length; ++i) {
+ comparators[i] = comparatorFactories[i].createBinaryComparator();
+ }
+ IOperatorNodePushable op = new IOperatorNodePushable() {
+ private List<ByteBuffer> buffers;
+
+ private final FrameTupleAccessor fta1 = new FrameTupleAccessor(ctx, recordDescriptors[0]);
+ private final FrameTupleAccessor fta2 = new FrameTupleAccessor(ctx, recordDescriptors[0]);
+
+ @Override
+ public void open() throws HyracksDataException {
+ buffers = new ArrayList<ByteBuffer>();
+ env.set(BUFFERS, buffers);
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ ByteBuffer copy = ctx.getResourceManager().allocateFrame();
+ FrameUtils.copy(buffer, copy);
+ buffers.add(copy);
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ FrameTupleAccessor accessor = new FrameTupleAccessor(ctx, recordDescriptors[0]);
+ int nBuffers = buffers.size();
+ int totalTCount = 0;
+ for (int i = 0; i < nBuffers; ++i) {
+ accessor.reset(buffers.get(i));
+ totalTCount += accessor.getTupleCount();
+ }
+ long[] tPointers = new long[totalTCount];
+ int ptr = 0;
+ for (int i = 0; i < nBuffers; ++i) {
+ accessor.reset(buffers.get(i));
+ int tCount = accessor.getTupleCount();
+ for (int j = 0; j < tCount; ++j) {
+ tPointers[ptr++] = (((long) i) << 32) + j;
+ }
+ }
+ if (tPointers.length > 0) {
+ sort(tPointers, 0, tPointers.length);
+ }
+ env.set(TPOINTERS, tPointers);
+ }
+
+ private void sort(long[] tPointers, int offset, int length) {
+ int m = offset + (length >> 1);
+ long v = tPointers[m];
+
+ int a = offset;
+ int b = a;
+ int c = offset + length - 1;
+ int d = c;
+ while (true) {
+ while (b <= c && compare(tPointers[b], v) <= 0) {
+ if (compare(tPointers[b], v) == 0) {
+ swap(tPointers, a++, b);
+ }
+ ++b;
+ }
+ while (c >= b && compare(tPointers[c], v) >= 0) {
+ if (compare(tPointers[c], v) == 0) {
+ swap(tPointers, c, d--);
+ }
+ --c;
+ }
+ if (b > c)
+ break;
+ swap(tPointers, b++, c--);
+ }
+
+ int s;
+ int n = offset + length;
+ s = Math.min(a - offset, b - a);
+ vecswap(tPointers, offset, b - s, s);
+ s = Math.min(d - c, n - d - 1);
+ vecswap(tPointers, b, n - s, s);
+
+ if ((s = b - a) > 1) {
+ sort(tPointers, offset, s);
+ }
+ if ((s = d - c) > 1) {
+ sort(tPointers, n - s, s);
+ }
+ }
+
+ private void swap(long x[], int a, int b) {
+ long t = x[a];
+ x[a] = x[b];
+ x[b] = t;
+ }
+
+ private void vecswap(long x[], int a, int b, int n) {
+ for (int i = 0; i < n; i++, a++, b++) {
+ swap(x, a, b);
+ }
+ }
+
+ private int compare(long tp1, long tp2) {
+ int i1 = (int) ((tp1 >> 32) & 0xffffffff);
+ int j1 = (int) (tp1 & 0xffffffff);
+ int i2 = (int) ((tp2 >> 32) & 0xffffffff);
+ int j2 = (int) (tp2 & 0xffffffff);
+ ByteBuffer buf1 = buffers.get(i1);
+ ByteBuffer buf2 = buffers.get(i2);
+ byte[] b1 = buf1.array();
+ byte[] b2 = buf2.array();
+ fta1.reset(buf1);
+ fta2.reset(buf2);
+ for (int f = 0; f < sortFields.length; ++f) {
+ int fIdx = sortFields[f];
+ int s1 = fta1.getTupleStartOffset(j1) + fta1.getFieldSlotsLength()
+ + fta1.getFieldStartOffset(j1, fIdx);
+ int l1 = fta1.getFieldEndOffset(j1, fIdx) - fta1.getFieldStartOffset(j1, fIdx);
+ int s2 = fta2.getTupleStartOffset(j2) + fta2.getFieldSlotsLength()
+ + fta2.getFieldStartOffset(j2, fIdx);
+ int l2 = fta2.getFieldEndOffset(j2, fIdx) - fta2.getFieldStartOffset(j2, fIdx);
+ int c = comparators[f].compare(b1, s1, l1, b2, s2, l2);
+ if (c != 0) {
+ return c;
+ }
+ }
+ return 0;
+ }
+
+ @Override
+ public void setFrameWriter(int index, IFrameWriter writer) {
+ throw new IllegalArgumentException();
+ }
+ };
+ return op;
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return false;
+ }
+ }
+
+ private class MergeActivity extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IOperatorDescriptor getOwner() {
+ return InMemorySortOperatorDescriptor.this;
+ }
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ return null;
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final HyracksContext ctx, JobPlan plan,
+ final IOperatorEnvironment env, int partition) {
+ IOperatorNodePushable op = new IOperatorNodePushable() {
+ private IFrameWriter writer;
+
+ @Override
+ public void open() throws HyracksDataException {
+ List<ByteBuffer> buffers = (List<ByteBuffer>) env.get(BUFFERS);
+ long[] tPointers = (long[]) env.get(TPOINTERS);
+ FrameTupleAccessor accessor = new FrameTupleAccessor(ctx, recordDescriptors[0]);
+ FrameTupleAppender appender = new FrameTupleAppender(ctx);
+ ByteBuffer outFrame = ctx.getResourceManager().allocateFrame();
+ writer.open();
+ appender.reset(outFrame, true);
+ for (int ptr = 0; ptr < tPointers.length; ++ptr) {
+ long tp = tPointers[ptr];
+ int i = (int) ((tp >> 32) & 0xffffffff);
+ int j = (int) (tp & 0xffffffff);
+ ByteBuffer buffer = buffers.get(i);
+ accessor.reset(buffer);
+ if (!appender.append(accessor, j)) {
+ flushFrame(outFrame);
+ appender.reset(outFrame, true);
+ if (!appender.append(accessor, j)) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+ if (appender.getTupleCount() > 0) {
+ flushFrame(outFrame);
+ }
+ writer.close();
+ env.set(BUFFERS, null);
+ env.set(TPOINTERS, null);
+ }
+
+ private void flushFrame(ByteBuffer frame) throws HyracksDataException {
+ frame.position(0);
+ frame.limit(frame.capacity());
+ writer.nextFrame(frame);
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ // do nothing
+ }
+
+ @Override
+ public void setFrameWriter(int index, IFrameWriter writer) {
+ if (index != 0) {
+ throw new IllegalArgumentException();
+ }
+ this.writer = writer;
+ }
+ };
+ return op;
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MToNHashPartitioningConnectorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MToNHashPartitioningConnectorDescriptor.java
new file mode 100644
index 0000000..937e0a4
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MToNHashPartitioningConnectorDescriptor.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import edu.uci.ics.hyracks.api.comm.IConnectionDemultiplexer;
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.IEndpointDataWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.comm.NonDeterministicFrameReader;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.AbstractConnectorDescriptor;
+
+public class MToNHashPartitioningConnectorDescriptor extends AbstractConnectorDescriptor {
+ private static final long serialVersionUID = 1L;
+ private ITuplePartitionComputerFactory tpcf;
+
+ public MToNHashPartitioningConnectorDescriptor(JobSpecification spec, ITuplePartitionComputerFactory tpcf) {
+ super(spec);
+ this.tpcf = tpcf;
+ }
+
+ @Override
+ public IFrameWriter createSendSideWriter(HyracksContext ctx, JobPlan plan, IEndpointDataWriterFactory edwFactory,
+ int index) throws HyracksDataException {
+ JobSpecification spec = plan.getJobSpecification();
+ final int consumerPartitionCount = spec.getConsumer(this).getPartitions().length;
+ final HashDataWriter hashWriter = new HashDataWriter(ctx, consumerPartitionCount, edwFactory, spec
+ .getConnectorRecordDescriptor(this), tpcf.createPartitioner());
+ return hashWriter;
+ }
+
+ @Override
+ public IFrameReader createReceiveSideReader(HyracksContext ctx, JobPlan plan, IConnectionDemultiplexer demux,
+ int index) throws HyracksDataException {
+ return new NonDeterministicFrameReader(ctx, demux);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MToNHashPartitioningMergingConnectorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MToNHashPartitioningMergingConnectorDescriptor.java
new file mode 100644
index 0000000..9f36730
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MToNHashPartitioningMergingConnectorDescriptor.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import edu.uci.ics.hyracks.api.comm.IConnectionDemultiplexer;
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.IEndpointDataWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.comm.SortMergeFrameReader;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.AbstractConnectorDescriptor;
+
+public class MToNHashPartitioningMergingConnectorDescriptor extends AbstractConnectorDescriptor {
+ private static final long serialVersionUID = 1L;
+
+ private final ITuplePartitionComputerFactory tpcf;
+ private final int[] sortFields;
+ private final IBinaryComparatorFactory[] comparatorFactories;
+
+ public MToNHashPartitioningMergingConnectorDescriptor(JobSpecification spec, ITuplePartitionComputerFactory tpcf,
+ int[] sortFields, IBinaryComparatorFactory[] comparatorFactories) {
+ super(spec);
+ this.tpcf = tpcf;
+ this.sortFields = sortFields;
+ this.comparatorFactories = comparatorFactories;
+ }
+
+ @Override
+ public IFrameWriter createSendSideWriter(HyracksContext ctx, JobPlan plan, IEndpointDataWriterFactory edwFactory,
+ int index) throws HyracksDataException {
+ JobSpecification spec = plan.getJobSpecification();
+ final int consumerPartitionCount = spec.getConsumer(this).getPartitions().length;
+ final HashDataWriter hashWriter = new HashDataWriter(ctx, consumerPartitionCount, edwFactory, spec
+ .getConnectorRecordDescriptor(this), tpcf.createPartitioner());
+ return hashWriter;
+ }
+
+ @Override
+ public IFrameReader createReceiveSideReader(HyracksContext ctx, JobPlan plan, IConnectionDemultiplexer demux, int index)
+ throws HyracksDataException {
+ IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
+ for(int i = 0; i < comparatorFactories.length; ++i) {
+ comparators[i] = comparatorFactories[i].createBinaryComparator();
+ }
+ JobSpecification spec = plan.getJobSpecification();
+ return new SortMergeFrameReader(ctx, demux, sortFields, comparators, spec.getConnectorRecordDescriptor(this));
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MToNRangePartitioningConnectorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MToNRangePartitioningConnectorDescriptor.java
new file mode 100644
index 0000000..f455f01
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MToNRangePartitioningConnectorDescriptor.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.comm.IConnectionDemultiplexer;
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.IEndpointDataWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.comm.NonDeterministicFrameReader;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.AbstractConnectorDescriptor;
+
+public class MToNRangePartitioningConnectorDescriptor extends AbstractConnectorDescriptor {
+ private static final long serialVersionUID = 1L;
+
+ private class RangeDataWriter implements IFrameWriter {
+ private final int consumerPartitionCount;
+ private final IFrameWriter[] epWriters;
+ private final FrameTupleAppender[] appenders;
+ private final FrameTupleAccessor tupleAccessor;
+
+ public RangeDataWriter(HyracksContext ctx, int consumerPartitionCount, IFrameWriter[] epWriters,
+ FrameTupleAppender[] appenders, RecordDescriptor recordDescriptor) {
+ this.consumerPartitionCount = consumerPartitionCount;
+ this.epWriters = epWriters;
+ this.appenders = appenders;
+ tupleAccessor = new FrameTupleAccessor(ctx, recordDescriptor);
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ for (int i = 0; i < epWriters.length; ++i) {
+ if (appenders[i].getTupleCount() > 0) {
+ flushFrame(appenders[i].getBuffer(), epWriters[i]);
+ }
+ epWriters[i].close();
+ }
+ }
+
+ private void flushFrame(ByteBuffer buffer, IFrameWriter frameWriter) throws HyracksDataException {
+ buffer.position(0);
+ buffer.limit(buffer.capacity());
+ frameWriter.nextFrame(buffer);
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ if (true) {
+ throw new UnsupportedOperationException();
+ }
+ tupleAccessor.reset(buffer);
+ int slotLength = tupleAccessor.getFieldSlotsLength();
+ int tupleCount = tupleAccessor.getTupleCount();
+ for (int i = 0; i < tupleCount; ++i) {
+ int startOffset = tupleAccessor.getTupleStartOffset(i);
+ int fStart = tupleAccessor.getFieldStartOffset(i, partitioningField);
+ int fEnd = tupleAccessor.getFieldEndOffset(i, partitioningField);
+ int h = 0;
+ FrameTupleAppender appender = appenders[h];
+ if (!appender.append(tupleAccessor, i)) {
+ ByteBuffer appenderBuffer = appender.getBuffer();
+ flushFrame(appenderBuffer, epWriters[h]);
+ appender.reset(appenderBuffer, true);
+ }
+ }
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ for (int i = 0; i < epWriters.length; ++i) {
+ epWriters[i].open();
+ appenders[i].reset(appenders[i].getBuffer(), true);
+ }
+ }
+ }
+
+ private final int partitioningField;
+ private final Object[] splitVector;
+
+ public MToNRangePartitioningConnectorDescriptor(JobSpecification spec, int partitioningField, Object[] splitVector) {
+ super(spec);
+ this.partitioningField = partitioningField;
+ this.splitVector = splitVector;
+ }
+
+ @Override
+ public IFrameWriter createSendSideWriter(HyracksContext ctx, JobPlan plan, IEndpointDataWriterFactory edwFactory,
+ int index) throws HyracksDataException {
+ JobSpecification spec = plan.getJobSpecification();
+ final int consumerPartitionCount = spec.getConsumer(this).getPartitions().length;
+ final IFrameWriter[] epWriters = new IFrameWriter[consumerPartitionCount];
+ final FrameTupleAppender[] appenders = new FrameTupleAppender[consumerPartitionCount];
+ for (int i = 0; i < consumerPartitionCount; ++i) {
+ try {
+ epWriters[i] = edwFactory.createFrameWriter(i);
+ appenders[i] = new FrameTupleAppender(ctx);
+ appenders[i].reset(ctx.getResourceManager().allocateFrame(), true);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ final RangeDataWriter rangeWriter = new RangeDataWriter(ctx, consumerPartitionCount, epWriters, appenders, spec
+ .getConnectorRecordDescriptor(this));
+ return rangeWriter;
+ }
+
+ @Override
+ public IFrameReader createReceiveSideReader(HyracksContext ctx, JobPlan plan, IConnectionDemultiplexer demux,
+ int index) throws HyracksDataException {
+ return new NonDeterministicFrameReader(ctx, demux);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MToNReplicatingConnectorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MToNReplicatingConnectorDescriptor.java
new file mode 100644
index 0000000..50d804a
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MToNReplicatingConnectorDescriptor.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.comm.IConnectionDemultiplexer;
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.IEndpointDataWriterFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.comm.NonDeterministicFrameReader;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.AbstractConnectorDescriptor;
+
+public class MToNReplicatingConnectorDescriptor extends AbstractConnectorDescriptor {
+ public MToNReplicatingConnectorDescriptor(JobSpecification spec) {
+ super(spec);
+ }
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IFrameWriter createSendSideWriter(HyracksContext ctx, JobPlan plan, IEndpointDataWriterFactory edwFactory,
+ int index) throws HyracksDataException {
+ JobSpecification spec = plan.getJobSpecification();
+ final int consumerPartitionCount = spec.getConsumer(this).getPartitions().length;
+ final IFrameWriter[] epWriters = new IFrameWriter[consumerPartitionCount];
+ for (int i = 0; i < consumerPartitionCount; ++i) {
+ epWriters[i] = edwFactory.createFrameWriter(i);
+ }
+ return new IFrameWriter() {
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ buffer.mark();
+ for (int i = 0; i < epWriters.length; ++i) {
+ if (i != 0) {
+ buffer.reset();
+ }
+ epWriters[i].nextFrame(buffer);
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ for (int i = 0; i < epWriters.length; ++i) {
+ epWriters[i].close();
+ }
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ for (int i = 0; i < epWriters.length; ++i) {
+ epWriters[i].open();
+ }
+ }
+ };
+ }
+
+ @Override
+ public IFrameReader createReceiveSideReader(HyracksContext ctx, JobPlan plan, IConnectionDemultiplexer demux,
+ int index) throws HyracksDataException {
+ return new NonDeterministicFrameReader(ctx, demux);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MapperOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MapperOperatorDescriptor.java
new file mode 100644
index 0000000..faad82c
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MapperOperatorDescriptor.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataWriter;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePullable;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.AbstractSingleActivityOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.base.IOpenableDataWriterOperator;
+import edu.uci.ics.hyracks.coreops.util.DeserializedOperatorNodePushable;
+
+public class MapperOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+ private class MapperOperator implements IOpenableDataWriterOperator {
+ private IMapper mapper;
+ private IOpenableDataWriter<Object[]> writer;
+
+ @Override
+ public void close() throws HyracksDataException {
+ // writer.writeData(null);
+ writer.close();
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ mapper = mapperFactory.createMapper();
+ writer.open();
+ }
+
+ @Override
+ public void setDataWriter(int index, IOpenableDataWriter<Object[]> writer) {
+ if (index != 0) {
+ throw new IllegalArgumentException();
+ }
+ this.writer = writer;
+ }
+
+ @Override
+ public void writeData(Object[] data) throws HyracksDataException {
+ mapper.map(data, writer);
+ }
+ }
+
+ private static final long serialVersionUID = 1L;
+
+ private final IMapperFactory mapperFactory;
+
+ public MapperOperatorDescriptor(JobSpecification spec, IMapperFactory mapperFactory,
+ RecordDescriptor recordDescriptor) {
+ super(spec, 1, 1);
+ this.mapperFactory = mapperFactory;
+ recordDescriptors[0] = recordDescriptor;
+ }
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ return new DeserializedOperatorNodePushable(ctx, new MapperOperator(), plan, getActivityNodeId());
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MaterializingOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MaterializingOperatorDescriptor.java
new file mode 100644
index 0000000..5e52989
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/MaterializingOperatorDescriptor.java
@@ -0,0 +1,214 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePullable;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.AbstractActivityNode;
+import edu.uci.ics.hyracks.coreops.base.AbstractOperatorDescriptor;
+
+public class MaterializingOperatorDescriptor extends AbstractOperatorDescriptor {
+ private static final long serialVersionUID = 1L;
+ protected static final String MATERIALIZED_FILE = "materialized-file";
+ protected static final String FRAME_COUNT = "frame-count";
+
+ public MaterializingOperatorDescriptor(JobSpecification spec, RecordDescriptor recordDescriptor) {
+ super(spec, 1, 1);
+ recordDescriptors[0] = recordDescriptor;
+ }
+
+ @Override
+ public void contributeTaskGraph(IActivityGraphBuilder builder) {
+ MaterializerActivityNode ma = new MaterializerActivityNode();
+ ReaderActivityNode ra = new ReaderActivityNode();
+
+ builder.addTask(ma);
+ builder.addSourceEdge(0, ma, 0);
+
+ builder.addTask(ra);
+ builder.addTargetEdge(0, ra, 0);
+
+ builder.addBlockingEdge(ma, ra);
+ }
+
+ private final class MaterializerActivityNode extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final HyracksContext ctx, JobPlan plan,
+ final IOperatorEnvironment env, int partition) {
+ return new IOperatorNodePushable() {
+ private FileChannel out;
+ private int frameCount;
+
+ @Override
+ public void open() throws HyracksDataException {
+ File outFile;
+ try {
+ outFile = ctx.getResourceManager().createFile("mat", ".dat");
+ out = new RandomAccessFile(outFile, "rw").getChannel();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ env.set(MATERIALIZED_FILE, outFile.getAbsolutePath());
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ ++frameCount;
+ buffer.position(0);
+ buffer.limit(buffer.capacity());
+ int rem = buffer.capacity();
+ while (rem > 0) {
+ int c;
+ try {
+ c = out.write(buffer);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ rem -= c;
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ try {
+ env.set(FRAME_COUNT, frameCount);
+ out.close();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void setFrameWriter(int index, IFrameWriter writer) {
+ throw new IllegalArgumentException();
+ }
+ };
+ }
+
+ @Override
+ public IOperatorDescriptor getOwner() {
+ return MaterializingOperatorDescriptor.this;
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+ }
+
+ private final class ReaderActivityNode extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final HyracksContext ctx, JobPlan plan,
+ final IOperatorEnvironment env, int partition) {
+ return new IOperatorNodePushable() {
+ private IFrameWriter writer;
+
+ @Override
+ public void open() throws HyracksDataException {
+ try {
+ File inFile = new File((String) env.get(MATERIALIZED_FILE));
+ int frameCount = (Integer) env.get(FRAME_COUNT);
+ FileChannel in = new RandomAccessFile(inFile, "r").getChannel();
+ ByteBuffer frame = ctx.getResourceManager().allocateFrame();
+ writer.open();
+ for (int i = 0; i < frameCount; ++i) {
+ frame.clear();
+ int rem = frame.capacity();
+ while (rem > 0) {
+ int c = in.read(frame);
+ rem -= c;
+ }
+ frame.flip();
+ writer.nextFrame(frame);
+ }
+ writer.close();
+ in.close();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ env.set(MATERIALIZED_FILE, null);
+ }
+
+ @Override
+ public void setFrameWriter(int index, IFrameWriter writer) {
+ if (index != 0) {
+ throw new IllegalArgumentException();
+ }
+ this.writer = writer;
+ }
+ };
+ }
+
+ @Override
+ public IOperatorDescriptor getOwner() {
+ return MaterializingOperatorDescriptor.this;
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/OneToOneConnectorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/OneToOneConnectorDescriptor.java
new file mode 100644
index 0000000..c46026a
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/OneToOneConnectorDescriptor.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import edu.uci.ics.hyracks.api.comm.IConnectionDemultiplexer;
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.IEndpointDataWriterFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.comm.NonDeterministicFrameReader;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.AbstractConnectorDescriptor;
+
+public class OneToOneConnectorDescriptor extends AbstractConnectorDescriptor {
+ private static final long serialVersionUID = 1L;
+
+ public OneToOneConnectorDescriptor(JobSpecification spec) {
+ super(spec);
+ }
+
+ @Override
+ public IFrameWriter createSendSideWriter(HyracksContext ctx, JobPlan plan, IEndpointDataWriterFactory edwFactory,
+ int index) throws HyracksDataException {
+ return edwFactory.createFrameWriter(index);
+ }
+
+ @Override
+ public IFrameReader createReceiveSideReader(HyracksContext ctx, JobPlan plan, IConnectionDemultiplexer demux,
+ int index) throws HyracksDataException {
+ return new NonDeterministicFrameReader(ctx, demux);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/PrinterOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/PrinterOperatorDescriptor.java
new file mode 100644
index 0000000..fd32245
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/PrinterOperatorDescriptor.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataWriter;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePullable;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.AbstractSingleActivityOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.base.IOpenableDataWriterOperator;
+import edu.uci.ics.hyracks.coreops.util.DeserializedOperatorNodePushable;
+
+public class PrinterOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+ private static final long serialVersionUID = 1L;
+
+ public PrinterOperatorDescriptor(JobSpecification spec) {
+ super(spec, 1, 0);
+ }
+
+ private class PrinterOperator implements IOpenableDataWriterOperator {
+ @Override
+ public void open() throws HyracksDataException {
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ }
+
+ @Override
+ public void writeData(Object[] data) throws HyracksDataException {
+ for (int i = 0; i < data.length; ++i) {
+ System.err.print(String.valueOf(data[i]));
+ System.err.print(", ");
+ }
+ System.err.println();
+ }
+
+ @Override
+ public void setDataWriter(int index, IOpenableDataWriter<Object[]> writer) {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ return new DeserializedOperatorNodePushable(ctx, new PrinterOperator(), plan, getActivityNodeId());
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/SplitVectorOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/SplitVectorOperatorDescriptor.java
new file mode 100644
index 0000000..feb4d2f
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/SplitVectorOperatorDescriptor.java
@@ -0,0 +1,185 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.dataflow.ActivityNodeId;
+import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder;
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataWriter;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePullable;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.AbstractActivityNode;
+import edu.uci.ics.hyracks.coreops.base.AbstractOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.base.IOpenableDataWriterOperator;
+import edu.uci.ics.hyracks.coreops.util.DeserializedOperatorNodePushable;
+
+public class SplitVectorOperatorDescriptor extends AbstractOperatorDescriptor {
+ private class CollectActivity extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public ActivityNodeId getActivityNodeId() {
+ return id;
+ }
+
+ @Override
+ public IOperatorDescriptor getOwner() {
+ return SplitVectorOperatorDescriptor.this;
+ }
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(HyracksContext ctx, JobPlan plan, final IOperatorEnvironment env,
+ int partition) {
+ IOpenableDataWriterOperator op = new IOpenableDataWriterOperator() {
+ private ArrayList<Object[]> buffer;
+
+ @Override
+ public void setDataWriter(int index, IOpenableDataWriter<Object[]> writer) {
+ throw new IllegalArgumentException();
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ buffer = new ArrayList<Object[]>();
+ env.set(BUFFER, buffer);
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+
+ }
+
+ @Override
+ public void writeData(Object[] data) throws HyracksDataException {
+ buffer.add(data);
+ }
+ };
+ return new DeserializedOperatorNodePushable(ctx, op, plan, getActivityNodeId());
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+ }
+
+ private class SplitActivity extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IOperatorDescriptor getOwner() {
+ return SplitVectorOperatorDescriptor.this;
+ }
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(HyracksContext ctx, JobPlan plan, final IOperatorEnvironment env,
+ int partition) {
+ IOpenableDataWriterOperator op = new IOpenableDataWriterOperator() {
+ private IOpenableDataWriter<Object[]> writer;
+
+ @Override
+ public void setDataWriter(int index, IOpenableDataWriter<Object[]> writer) {
+ if (index != 0) {
+ throw new IllegalArgumentException();
+ }
+ this.writer = writer;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ }
+
+ @Override
+ public void writeData(Object[] data) throws HyracksDataException {
+ List<Object[]> buffer = (List<Object[]>) env.get(BUFFER);
+ int n = buffer.size();
+ int step = (int) Math.floor(n / (float) splits);
+ writer.open();
+ for (int i = 0; i < splits; ++i) {
+ writer.writeData(buffer.get(step * (i + 1) - 1));
+ }
+ writer.close();
+ }
+ };
+ return new DeserializedOperatorNodePushable(ctx, op, plan, getActivityNodeId());
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+ }
+
+ private static final String BUFFER = "buffer";
+
+ private static final long serialVersionUID = 1L;
+
+ private final int splits;
+
+ public SplitVectorOperatorDescriptor(JobSpecification spec, int splits, RecordDescriptor recordDescriptor) {
+ super(spec, 1, 1);
+ this.splits = splits;
+ recordDescriptors[0] = recordDescriptor;
+ }
+
+ @Override
+ public void contributeTaskGraph(IActivityGraphBuilder builder) {
+ CollectActivity ca = new CollectActivity();
+ SplitActivity sa = new SplitActivity();
+
+ builder.addTask(ca);
+ builder.addSourceEdge(0, ca, 0);
+
+ builder.addTask(sa);
+ builder.addTargetEdge(0, sa, 0);
+
+ builder.addBlockingEdge(ca, sa);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/aggregators/ReflectionBasedTupleAggregatorFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/aggregators/ReflectionBasedTupleAggregatorFactory.java
new file mode 100644
index 0000000..f649e48
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/aggregators/ReflectionBasedTupleAggregatorFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.aggregators;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.coreops.ITupleAggregator;
+import edu.uci.ics.hyracks.coreops.ITupleAggregatorFactory;
+
+public class ReflectionBasedTupleAggregatorFactory implements
+ ITupleAggregatorFactory {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ private final Class<? extends ITupleAggregator> aggregatorClass;
+
+ public ReflectionBasedTupleAggregatorFactory(
+ Class<? extends ITupleAggregator> aggregatorClass) {
+ this.aggregatorClass = aggregatorClass;
+ }
+
+ @Override
+ public ITupleAggregator createTupleAggregator() throws HyracksDataException {
+ try {
+ return aggregatorClass.newInstance();
+ } catch (InstantiationException e) {
+ throw new HyracksDataException(e);
+ } catch (IllegalAccessException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/aggregators/SumGroupAggregator.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/aggregators/SumGroupAggregator.java
new file mode 100644
index 0000000..f489a55
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/aggregators/SumGroupAggregator.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.aggregators;
+
+import edu.uci.ics.hyracks.api.dataflow.IDataReader;
+import edu.uci.ics.hyracks.api.dataflow.IDataWriter;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.coreops.group.IGroupAggregator;
+
+public class SumGroupAggregator implements IGroupAggregator {
+ private static final long serialVersionUID = 1L;
+ private int keyIdx;
+
+ public SumGroupAggregator(int keyIdx) {
+ this.keyIdx = keyIdx;
+ }
+
+ @Override
+ public void aggregate(IDataReader<Object[]> reader, IDataWriter<Object[]> writer) throws HyracksDataException {
+ String key = "";
+ Object[] data;
+ int count = 0;
+ while ((data = reader.readData()) != null) {
+ key = (String) data[keyIdx];
+ ++count;
+ }
+ writer.writeData(new Object[] { key, count });
+ }
+
+ @Override
+ public void close() throws Exception {
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/aggregators/SumStringGroupAggregator.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/aggregators/SumStringGroupAggregator.java
new file mode 100644
index 0000000..7c774d3
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/aggregators/SumStringGroupAggregator.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.aggregators;
+
+import edu.uci.ics.hyracks.api.dataflow.IDataReader;
+import edu.uci.ics.hyracks.api.dataflow.IDataWriter;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.coreops.group.IGroupAggregator;
+
+public class SumStringGroupAggregator implements IGroupAggregator {
+ private static final long serialVersionUID = 1L;
+ private int keyIdx;
+
+ public SumStringGroupAggregator(int keyIdx) {
+ this.keyIdx = keyIdx;
+ }
+
+ @Override
+ public void aggregate(IDataReader<Object[]> reader, IDataWriter<Object[]> writer) throws HyracksDataException {
+ String key = "";
+ Object[] data;
+ int count = 0;
+ while ((data = reader.readData()) != null) {
+ key = (String) data[keyIdx];
+ ++count;
+ }
+ writer.writeData(new Object[] { key, String.valueOf(count) });
+ }
+
+ @Override
+ public void close() throws Exception {
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/aggregators/SumTupleAggregator.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/aggregators/SumTupleAggregator.java
new file mode 100644
index 0000000..cfa512e
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/aggregators/SumTupleAggregator.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.aggregators;
+
+import edu.uci.ics.hyracks.api.dataflow.IDataWriter;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.coreops.ITupleAggregator;
+
+public class SumTupleAggregator implements ITupleAggregator {
+
+ private Object key;
+ private int count;
+
+ @Override
+ public void add(Object[] data) {
+ count++;
+ }
+
+ @Override
+ public void init(Object[] data) {
+ key = data[0];
+ count = 0;
+ }
+
+ @Override
+ public void write(IDataWriter<Object[]> writer) throws HyracksDataException {
+ writer.writeData(new Object[] { key, count });
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/base/AbstractActivityNode.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/base/AbstractActivityNode.java
new file mode 100644
index 0000000..b2e39e8
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/base/AbstractActivityNode.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.base;
+
+import java.util.UUID;
+
+import edu.uci.ics.hyracks.api.dataflow.ActivityNodeId;
+import edu.uci.ics.hyracks.api.dataflow.IActivityNode;
+
+public abstract class AbstractActivityNode implements IActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ protected final ActivityNodeId id;
+
+ public AbstractActivityNode() {
+ this.id = new ActivityNodeId(getOwner().getOperatorId(), UUID.randomUUID());
+ }
+
+ @Override
+ public ActivityNodeId getActivityNodeId() {
+ return id;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/base/AbstractConnectorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/base/AbstractConnectorDescriptor.java
new file mode 100644
index 0000000..72dbcbf
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/base/AbstractConnectorDescriptor.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.base;
+
+import java.util.UUID;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public abstract class AbstractConnectorDescriptor implements IConnectorDescriptor {
+ private static final long serialVersionUID = 1L;
+ protected final ConnectorDescriptorId id;
+
+ public AbstractConnectorDescriptor(JobSpecification spec) {
+ this.id = new ConnectorDescriptorId(UUID.randomUUID());
+ spec.getConnectorMap().put(id, this);
+ }
+
+ public ConnectorDescriptorId getConnectorId() {
+ return id;
+ }
+
+ @Override
+ public JSONObject toJSON() throws JSONException {
+ JSONObject jconn = new JSONObject();
+
+ jconn.put("type", "connector");
+ jconn.put("id", getConnectorId().getId().toString());
+ jconn.put("java-class", getClass().getName());
+
+ return jconn;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/base/AbstractOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/base/AbstractOperatorDescriptor.java
new file mode 100644
index 0000000..b411020
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/base/AbstractOperatorDescriptor.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.base;
+
+import java.util.UUID;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.api.constraints.PartitionConstraint;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public abstract class AbstractOperatorDescriptor implements IOperatorDescriptor {
+ private static final long serialVersionUID = 1L;
+
+ protected final OperatorDescriptorId odId;
+
+ protected PartitionConstraint partitionConstraint;
+
+ protected String[] partitions;
+
+ protected final RecordDescriptor[] recordDescriptors;
+
+ protected final int inputArity;
+
+ protected final int outputArity;
+
+ public AbstractOperatorDescriptor(JobSpecification spec, int inputArity, int outputArity) {
+ odId = new OperatorDescriptorId(UUID.randomUUID());
+ this.inputArity = inputArity;
+ this.outputArity = outputArity;
+ recordDescriptors = new RecordDescriptor[outputArity];
+ spec.getOperatorMap().put(getOperatorId(), this);
+ }
+
+ @Override
+ public final OperatorDescriptorId getOperatorId() {
+ return odId;
+ }
+
+ @Override
+ public int getInputArity() {
+ return inputArity;
+ }
+
+ @Override
+ public int getOutputArity() {
+ return outputArity;
+ }
+
+ @Override
+ public PartitionConstraint getPartitionConstraint() {
+ return partitionConstraint;
+ }
+
+ @Override
+ public void setPartitionConstraint(PartitionConstraint partitionConstraint) {
+ this.partitionConstraint = partitionConstraint;
+ }
+
+ @Override
+ public String[] getPartitions() {
+ return partitions;
+ }
+
+ @Override
+ public void setPartitions(String[] partitions) {
+ this.partitions = partitions;
+ }
+
+ @Override
+ public RecordDescriptor[] getOutputRecordDescriptors() {
+ return recordDescriptors;
+ }
+
+ @Override
+ public JSONObject toJSON() throws JSONException {
+ JSONObject jop = new JSONObject();
+ jop.put("type", "operator");
+ jop.put("id", getOperatorId().getId().toString());
+ jop.put("java-class", getClass().getName());
+ jop.put("in-arity", getInputArity());
+ jop.put("out-arity", getOutputArity());
+ return jop;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/base/AbstractSingleActivityOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/base/AbstractSingleActivityOperatorDescriptor.java
new file mode 100644
index 0000000..5843f10
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/base/AbstractSingleActivityOperatorDescriptor.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.base;
+
+import java.util.UUID;
+
+import edu.uci.ics.hyracks.api.dataflow.ActivityNodeId;
+import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder;
+import edu.uci.ics.hyracks.api.dataflow.IActivityNode;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public abstract class AbstractSingleActivityOperatorDescriptor extends AbstractOperatorDescriptor implements
+ IActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ protected final ActivityNodeId activityNodeId;
+
+ public AbstractSingleActivityOperatorDescriptor(JobSpecification spec, int inputArity, int outputArity) {
+ super(spec, inputArity, outputArity);
+ activityNodeId = new ActivityNodeId(odId, UUID.randomUUID());
+ }
+
+ @Override
+ public ActivityNodeId getActivityNodeId() {
+ return activityNodeId;
+ }
+
+ @Override
+ public final IOperatorDescriptor getOwner() {
+ return this;
+ }
+
+ @Override
+ public final void contributeTaskGraph(IActivityGraphBuilder builder) {
+ builder.addTask(this);
+ for (int i = 0; i < getInputArity(); ++i) {
+ builder.addSourceEdge(i, this, i);
+ }
+ for (int i = 0; i < getOutputArity(); ++i) {
+ builder.addTargetEdge(i, this, i);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/base/IOpenableDataWriterOperator.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/base/IOpenableDataWriterOperator.java
new file mode 100644
index 0000000..e6500d6
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/base/IOpenableDataWriterOperator.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.base;
+
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataWriter;
+
+public interface IOpenableDataWriterOperator extends IOpenableDataWriter<Object[]> {
+ public void setDataWriter(int index, IOpenableDataWriter<Object[]> writer);
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/FloatBinaryComparatorFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/FloatBinaryComparatorFactory.java
new file mode 100644
index 0000000..353d41d
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/FloatBinaryComparatorFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+
+public class FloatBinaryComparatorFactory implements IBinaryComparatorFactory {
+ private static final long serialVersionUID = 1L;
+
+ public static final FloatBinaryComparatorFactory INSTANCE = new FloatBinaryComparatorFactory();
+
+ private FloatBinaryComparatorFactory() {
+ }
+
+ @Override
+ public IBinaryComparator createBinaryComparator() {
+ return new IBinaryComparator() {
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ return Float.compare(FloatSerializerDeserializer.getFloat(b1, s1), FloatSerializerDeserializer
+ .getFloat(b2, s2));
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/FloatBinaryHashFunctionFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/FloatBinaryHashFunctionFactory.java
new file mode 100644
index 0000000..071ac70
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/FloatBinaryHashFunctionFactory.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunction;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+
+public class FloatBinaryHashFunctionFactory implements IBinaryHashFunctionFactory {
+ private static final long serialVersionUID = 1L;
+
+ public static final FloatBinaryHashFunctionFactory INSTANCE = new FloatBinaryHashFunctionFactory();
+
+ private FloatBinaryHashFunctionFactory() {
+ }
+
+ @Override
+ public IBinaryHashFunction createBinaryHashFunction() {
+ return new IBinaryHashFunction() {
+ @Override
+ public int hash(byte[] bytes, int offset, int length) {
+ return FloatSerializerDeserializer.getIntBits(bytes, offset);
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/FloatComparatorFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/FloatComparatorFactory.java
new file mode 100644
index 0000000..9cbf499
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/FloatComparatorFactory.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IComparatorFactory;
+
+public class FloatComparatorFactory implements IComparatorFactory<Float> {
+ private static final long serialVersionUID = 1L;
+
+ public static final FloatComparatorFactory INSTANCE = new FloatComparatorFactory();
+
+ private FloatComparatorFactory() {
+ }
+
+ @Override
+ public IComparator<Float> createComparator() {
+ return new IComparator<Float>() {
+ @Override
+ public int compare(Float o1, Float o2) {
+ return o1.compareTo(o2);
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/FloatHashFunctionFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/FloatHashFunctionFactory.java
new file mode 100644
index 0000000..a2831e8
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/FloatHashFunctionFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IHashFunction;
+import edu.uci.ics.hyracks.api.dataflow.value.IHashFunctionFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public class FloatHashFunctionFactory implements IHashFunctionFactory<Float> {
+ private static final long serialVersionUID = 1L;
+
+ public static final FloatHashFunctionFactory INSTANCE = new FloatHashFunctionFactory();
+
+ private FloatHashFunctionFactory() {
+ }
+
+ @Override
+ public IHashFunction<Float> createHashFunction() {
+ return new IHashFunction<Float>() {
+ @Override
+ public int hash(Float o) throws HyracksDataException {
+ return o.hashCode();
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/FloatSerializerDeserializer.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/FloatSerializerDeserializer.java
new file mode 100644
index 0000000..855da0e
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/FloatSerializerDeserializer.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public class FloatSerializerDeserializer implements ISerializerDeserializer<Float> {
+ private static final long serialVersionUID = 1L;
+
+ public static final FloatSerializerDeserializer INSTANCE = new FloatSerializerDeserializer();
+
+ private FloatSerializerDeserializer() {
+ }
+
+ @Override
+ public Float deserialize(DataInput in) throws HyracksDataException {
+ try {
+ return in.readFloat();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void serialize(Float instance, DataOutput out) throws HyracksDataException {
+ try {
+ out.writeFloat(instance.floatValue());
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ static float getFloat(byte[] bytes, int offset) {
+ return Float.intBitsToFloat(getIntBits(bytes, offset));
+ }
+
+ public static int getIntBits(byte[] bytes, int offset) {
+ return IntegerSerializerDeserializer.getInt(bytes, offset);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntArraySerializerDeserializer.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntArraySerializerDeserializer.java
new file mode 100644
index 0000000..410a6b9
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntArraySerializerDeserializer.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public class IntArraySerializerDeserializer implements ISerializerDeserializer<int[]> {
+ private static final long serialVersionUID = 1L;
+
+ public static final IntArraySerializerDeserializer INSTANCE = new IntArraySerializerDeserializer();
+
+ private IntArraySerializerDeserializer() {
+ }
+
+ @Override
+ public int[] deserialize(DataInput in) throws HyracksDataException {
+ try {
+ int len = in.readInt();
+ int[] array = new int[len];
+ for (int i = 0; i < array.length; ++i) {
+ array[i] = in.readInt();
+ }
+ return array;
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void serialize(int[] instance, DataOutput out) throws HyracksDataException {
+ try {
+ out.writeInt(instance.length);
+ for (int i = 0; i < instance.length; ++i) {
+ out.writeInt(instance[i]);
+ }
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntegerBinaryComparatorFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntegerBinaryComparatorFactory.java
new file mode 100644
index 0000000..f020909
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntegerBinaryComparatorFactory.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+
+public class IntegerBinaryComparatorFactory implements IBinaryComparatorFactory {
+ private static final long serialVersionUID = 1L;
+
+ public static final IntegerBinaryComparatorFactory INSTANCE = new IntegerBinaryComparatorFactory();
+
+ private IntegerBinaryComparatorFactory() {
+ }
+
+ @Override
+ public IBinaryComparator createBinaryComparator() {
+ return new IBinaryComparator() {
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ int v1 = IntegerSerializerDeserializer.getInt(b1, s1);
+ int v2 = IntegerSerializerDeserializer.getInt(b2, s2);
+ return v1 < v2 ? -1 : (v1 > v2 ? 1 : 0);
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntegerBinaryHashFunctionFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntegerBinaryHashFunctionFactory.java
new file mode 100644
index 0000000..38b51a2
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntegerBinaryHashFunctionFactory.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunction;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+
+public class IntegerBinaryHashFunctionFactory implements IBinaryHashFunctionFactory {
+ private static final long serialVersionUID = 1L;
+
+ public static final IntegerBinaryHashFunctionFactory INSTANCE = new IntegerBinaryHashFunctionFactory();
+
+ private IntegerBinaryHashFunctionFactory() {
+ }
+
+ @Override
+ public IBinaryHashFunction createBinaryHashFunction() {
+ return new IBinaryHashFunction() {
+ @Override
+ public int hash(byte[] bytes, int offset, int length) {
+ return IntegerSerializerDeserializer.getInt(bytes, offset);
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntegerComparatorFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntegerComparatorFactory.java
new file mode 100644
index 0000000..9f4394b
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntegerComparatorFactory.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IComparatorFactory;
+
+public class IntegerComparatorFactory implements IComparatorFactory<Integer> {
+ private static final long serialVersionUID = 1L;
+
+ public static final IntegerComparatorFactory INSTANCE = new IntegerComparatorFactory();
+
+ private IntegerComparatorFactory() {
+ }
+
+ @Override
+ public IComparator<Integer> createComparator() {
+ return new IComparator<Integer>() {
+ @Override
+ public int compare(Integer o1, Integer o2) {
+ return o1.compareTo(o2);
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntegerHashFunctionFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntegerHashFunctionFactory.java
new file mode 100644
index 0000000..5785084
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntegerHashFunctionFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IHashFunction;
+import edu.uci.ics.hyracks.api.dataflow.value.IHashFunctionFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public class IntegerHashFunctionFactory implements IHashFunctionFactory<Integer> {
+ private static final long serialVersionUID = 1L;
+
+ public static final IntegerHashFunctionFactory INSTANCE = new IntegerHashFunctionFactory();
+
+ private IntegerHashFunctionFactory() {
+ }
+
+ @Override
+ public IHashFunction<Integer> createHashFunction() {
+ return new IHashFunction<Integer>() {
+ @Override
+ public int hash(Integer o) throws HyracksDataException {
+ return o.hashCode();
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntegerSerializerDeserializer.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntegerSerializerDeserializer.java
new file mode 100644
index 0000000..74de8d3
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/IntegerSerializerDeserializer.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public class IntegerSerializerDeserializer implements ISerializerDeserializer<Integer> {
+ private static final long serialVersionUID = 1L;
+
+ public static final IntegerSerializerDeserializer INSTANCE = new IntegerSerializerDeserializer();
+
+ private IntegerSerializerDeserializer() {
+ }
+
+ @Override
+ public Integer deserialize(DataInput in) throws HyracksDataException {
+ try {
+ return in.readInt();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void serialize(Integer instance, DataOutput out) throws HyracksDataException {
+ try {
+ out.writeInt(instance.intValue());
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ static int getInt(byte[] bytes, int offset) {
+ return ((bytes[offset] & 0xff) << 24) + ((bytes[offset + 1] & 0xff) << 16) + ((bytes[offset + 2] & 0xff) << 8)
+ + ((bytes[offset + 3] & 0xff) << 0);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringBinaryComparatorFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringBinaryComparatorFactory.java
new file mode 100644
index 0000000..54212b6
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringBinaryComparatorFactory.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+
+public class StringBinaryComparatorFactory implements IBinaryComparatorFactory {
+ private static final long serialVersionUID = 1L;
+
+ public static final StringBinaryComparatorFactory INSTANCE = new StringBinaryComparatorFactory();
+
+ private StringBinaryComparatorFactory() {
+ }
+
+ @Override
+ public IBinaryComparator createBinaryComparator() {
+ return new IBinaryComparator() {
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ int utflen1 = StringUtils.getUTFLen(b1, s1);
+ int utflen2 = StringUtils.getUTFLen(b2, s2);
+
+ int c1 = 0;
+ int c2 = 0;
+
+ int s1Start = s1 + 2;
+ int s2Start = s2 + 2;
+
+ while (c1 < utflen1 && c2 < utflen2) {
+ char ch1 = StringUtils.charAt(b1, s1Start + c1);
+ char ch2 = StringUtils.charAt(b2, s2Start + c2);
+
+ if (ch1 != ch2) {
+ return ch1 - ch2;
+ }
+ c1 += StringUtils.charSize(b1, s1Start + c1);
+ c2 += StringUtils.charSize(b2, s2Start + c2);
+ }
+ return utflen1 - utflen2;
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringBinaryHashFunctionFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringBinaryHashFunctionFactory.java
new file mode 100644
index 0000000..3185442
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringBinaryHashFunctionFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunction;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+
+public class StringBinaryHashFunctionFactory implements IBinaryHashFunctionFactory {
+ public static final StringBinaryHashFunctionFactory INSTANCE = new StringBinaryHashFunctionFactory();
+
+ private static final long serialVersionUID = 1L;
+
+ private StringBinaryHashFunctionFactory() {
+ }
+
+ @Override
+ public IBinaryHashFunction createBinaryHashFunction() {
+ return new IBinaryHashFunction() {
+ @Override
+ public int hash(byte[] bytes, int offset, int length) {
+ int h = 0;
+ int utflen = StringUtils.getUTFLen(bytes, offset);
+ int sStart = offset + 2;
+ int c = 0;
+
+ while (c < utflen) {
+ char ch = StringUtils.charAt(bytes, sStart + c);
+ h = 31 * h + ch;
+ c += StringUtils.charSize(bytes, sStart + c);
+ }
+ return h;
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringComparatorFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringComparatorFactory.java
new file mode 100644
index 0000000..d3cbbff
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringComparatorFactory.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IComparatorFactory;
+
+public class StringComparatorFactory implements IComparatorFactory<String> {
+ private static final long serialVersionUID = 1L;
+
+ public static final StringComparatorFactory INSTANCE = new StringComparatorFactory();
+
+ private StringComparatorFactory() {
+ }
+
+ @Override
+ public IComparator<String> createComparator() {
+ return new IComparator<String>() {
+ @Override
+ public int compare(String o1, String o2) {
+ return o1.compareTo(o2);
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringHashFunctionFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringHashFunctionFactory.java
new file mode 100644
index 0000000..c488138
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringHashFunctionFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IHashFunction;
+import edu.uci.ics.hyracks.api.dataflow.value.IHashFunctionFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public class StringHashFunctionFactory implements IHashFunctionFactory<String> {
+ private static final long serialVersionUID = 1L;
+
+ public static final StringHashFunctionFactory INSTANCE = new StringHashFunctionFactory();
+
+ private StringHashFunctionFactory() {
+ }
+
+ @Override
+ public IHashFunction<String> createHashFunction() {
+ return new IHashFunction<String>() {
+ @Override
+ public int hash(String o) throws HyracksDataException {
+ return o.hashCode();
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringSerializerDeserializer.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringSerializerDeserializer.java
new file mode 100644
index 0000000..7342f1e
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringSerializerDeserializer.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public class StringSerializerDeserializer implements ISerializerDeserializer<String> {
+ public static final StringSerializerDeserializer INSTANCE = new StringSerializerDeserializer();
+
+ private static final long serialVersionUID = 1L;
+
+ private StringSerializerDeserializer() {
+ }
+
+ @Override
+ public String deserialize(DataInput in) throws HyracksDataException {
+ try {
+ return in.readUTF();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void serialize(String instance, DataOutput out) throws HyracksDataException {
+ try {
+ out.writeUTF(instance);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringUtils.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringUtils.java
new file mode 100644
index 0000000..36a7ed6
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/data/StringUtils.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.data;
+
+public class StringUtils {
+ public static int charSize(byte[] b, int s) {
+ int c = (int) b[s] & 0xff;
+ switch (c >> 4) {
+ case 0:
+ case 1:
+ case 2:
+ case 3:
+ case 4:
+ case 5:
+ case 6:
+ case 7:
+ return 1;
+
+ case 12:
+ case 13:
+ return 2;
+
+ case 14:
+ return 3;
+ }
+ throw new IllegalStateException();
+ }
+
+ public static char charAt(byte[] b, int s) {
+ int c = (int) b[s] & 0xff;
+ switch (c >> 4) {
+ case 0:
+ case 1:
+ case 2:
+ case 3:
+ case 4:
+ case 5:
+ case 6:
+ case 7:
+ return (char) c;
+
+ case 12:
+ case 13:
+ return (char) (((c & 0x1F) << 6) | (((int) b[s + 1]) & 0x3F));
+
+ case 14:
+ return (char) (((c & 0x0F) << 12) | ((((int) b[s + 1]) & 0x3F) << 6) | ((((int) b[s + 2]) & 0x3F) << 0));
+
+ default:
+ throw new IllegalArgumentException();
+ }
+ }
+
+ public static int getUTFLen(byte[] b, int s) {
+ return ((b[s] & 0xff) << 8) + ((b[s + 1] & 0xff) << 0);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/AbstractFileScanOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/AbstractFileScanOperatorDescriptor.java
new file mode 100644
index 0000000..82e805b
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/AbstractFileScanOperatorDescriptor.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.file;
+
+import java.io.File;
+
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataWriter;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePullable;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.AbstractSingleActivityOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.base.IOpenableDataWriterOperator;
+import edu.uci.ics.hyracks.coreops.util.DeserializedOperatorNodePushable;
+
+public abstract class AbstractFileScanOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+ private static final long serialVersionUID = 1L;
+
+ protected FileSplit[] splits;
+
+ public AbstractFileScanOperatorDescriptor(JobSpecification spec, FileSplit[] splits,
+ RecordDescriptor recordDescriptor) {
+ super(spec, 0, 1);
+ recordDescriptors[0] = recordDescriptor;
+ this.splits = splits;
+ }
+
+ protected abstract IRecordReader createRecordReader(File file, RecordDescriptor desc) throws Exception;
+
+ protected abstract void configure() throws Exception;
+
+ protected class FileScanOperator implements IOpenableDataWriterOperator {
+ private IOpenableDataWriter<Object[]> writer;
+ private int index;
+
+ FileScanOperator(int index) {
+ this.index = index;
+ }
+
+ @Override
+ public void setDataWriter(int index, IOpenableDataWriter<Object[]> writer) {
+ if (index != 0) {
+ throw new IndexOutOfBoundsException("Invalid index: " + index);
+ }
+ this.writer = writer;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ FileSplit split = splits[index];
+ RecordDescriptor desc = recordDescriptors[0];
+ try {
+ IRecordReader reader = createRecordReader(split.getLocalFile(), desc);
+ if (desc == null) {
+ desc = recordDescriptors[0];
+ }
+ writer.open();
+ try {
+ while (true) {
+ Object[] record = new Object[desc.getFields().length];
+ if (!reader.read(record)) {
+ break;
+ }
+ writer.writeData(record);
+ }
+ } finally {
+ reader.close();
+ writer.close();
+ }
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ // do nothing
+ }
+
+ @Override
+ public void writeData(Object[] data) throws HyracksDataException {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ return new DeserializedOperatorNodePushable(ctx, new FileScanOperator(partition), plan, getActivityNodeId());
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/AbstractFileWriteOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/AbstractFileWriteOperatorDescriptor.java
new file mode 100644
index 0000000..5bf6a86
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/AbstractFileWriteOperatorDescriptor.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.file;
+
+import java.io.File;
+
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataWriter;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePullable;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.AbstractSingleActivityOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.base.IOpenableDataWriterOperator;
+import edu.uci.ics.hyracks.coreops.util.DeserializedOperatorNodePushable;
+
+public abstract class AbstractFileWriteOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+ protected class FileWriteOperator implements IOpenableDataWriterOperator {
+ private int index;
+ private IRecordWriter writer;
+
+ FileWriteOperator(int index) {
+ this.index = index;
+ }
+
+ @Override
+ public void setDataWriter(int index, IOpenableDataWriter<Object[]> writer) {
+ throw new IllegalArgumentException();
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ FileSplit split = splits[index];
+ try {
+ writer = createRecordWriter(split.getLocalFile(), index);
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ writer.close();
+ }
+
+ @Override
+ public void writeData(Object[] data) throws HyracksDataException {
+ try {
+ writer.write(data);
+
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+
+ private static final long serialVersionUID = 1L;
+
+ protected FileSplit[] splits;
+
+ public FileSplit[] getSplits() {
+ return splits;
+ }
+
+ public void setSplits(FileSplit[] splits) {
+ this.splits = splits;
+ }
+
+ public AbstractFileWriteOperatorDescriptor(JobSpecification spec, FileSplit[] splits) {
+ super(spec, 1, 0);
+ this.splits = splits;
+ }
+
+ protected abstract IRecordWriter createRecordWriter(File file, int index) throws Exception;
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ return new DeserializedOperatorNodePushable(ctx, new FileWriteOperator(partition), plan, getActivityNodeId());
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/CSVFileScanOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/CSVFileScanOperatorDescriptor.java
new file mode 100644
index 0000000..efcb8c8
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/CSVFileScanOperatorDescriptor.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.file;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class CSVFileScanOperatorDescriptor extends
+ AbstractFileScanOperatorDescriptor {
+ private static class CSVRecordReaderImpl implements IRecordReader {
+ private final BufferedReader in;
+ private final char separator;
+ private final String quotes;
+
+ CSVRecordReaderImpl(File file, RecordDescriptor desc, char separator,
+ String quotes) throws Exception {
+ in = new BufferedReader(new InputStreamReader(new FileInputStream(
+ file)));
+ this.separator = separator;
+ this.quotes = quotes;
+ }
+
+ @Override
+ public void close() {
+ try {
+ in.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public boolean read(Object[] record) throws Exception {
+ String line = in.readLine();
+ if (line == null) {
+ return false;
+ }
+ int fid = 0;
+ char[] chars = line.toCharArray();
+ int i = 0;
+ boolean insideQuote = false;
+ char quoteChar = 0;
+ int partStart = 0;
+ boolean skipNext = false;
+ while (fid < record.length && i < chars.length) {
+ char c = chars[i];
+ if (!skipNext) {
+ if (quotes.indexOf(c) >= 0) {
+ if (insideQuote) {
+ if (quoteChar == c) {
+ insideQuote = false;
+ }
+ } else {
+ insideQuote = true;
+ quoteChar = c;
+ }
+ } else if (c == separator) {
+ if (!insideQuote) {
+ record[fid++] = String.valueOf(chars, partStart, i
+ - partStart);
+ partStart = i + 1;
+ }
+ } else if (c == '\\') {
+ skipNext = true;
+ }
+ } else {
+ skipNext = false;
+ }
+ ++i;
+ }
+ if (fid < record.length) {
+ record[fid] = String.valueOf(chars, partStart, i - partStart);
+ }
+ return true;
+ }
+ }
+
+ private static final long serialVersionUID = 1L;
+
+ private final char separator;
+ private final String quotes;
+
+ public CSVFileScanOperatorDescriptor(JobSpecification spec,
+ FileSplit[] splits, RecordDescriptor recordDescriptor) {
+ this(spec, splits, recordDescriptor, ',', "'\"");
+ }
+
+ public CSVFileScanOperatorDescriptor(JobSpecification spec,
+ FileSplit[] splits, RecordDescriptor recordDescriptor,
+ char separator, String quotes) {
+ super(spec, splits, recordDescriptor);
+ this.separator = separator;
+ this.quotes = quotes;
+ }
+
+ @Override
+ protected IRecordReader createRecordReader(File file, RecordDescriptor desc)
+ throws Exception {
+ return new CSVRecordReaderImpl(file, desc, separator, quotes);
+ }
+
+ @Override
+ protected void configure() throws Exception {
+ // currently a no-op, but is meant to initialize , if required before it is asked
+ // to create a record reader
+ // this is executed at the node and is useful for operators that could not be
+ // initialized from the client completely, because of lack of information specific
+ // to the node where the operator gets executed.
+
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/FileSplit.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/FileSplit.java
new file mode 100644
index 0000000..361afca
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/FileSplit.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.file;
+
+import java.io.File;
+import java.io.Serializable;
+
+public class FileSplit implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final String nodeName;
+
+ private final File file;
+
+ public FileSplit(String nodeName, File file) {
+ this.nodeName = nodeName;
+ this.file = file;
+ }
+
+ public String getNodeName() {
+ return nodeName;
+ }
+
+ public File getLocalFile() {
+ return file;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/IRecordReader.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/IRecordReader.java
new file mode 100644
index 0000000..607c637
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/IRecordReader.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.file;
+
+public interface IRecordReader {
+
+ public boolean read(Object[] record) throws Exception;
+
+ public void close();
+
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/IRecordWriter.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/IRecordWriter.java
new file mode 100644
index 0000000..a270f86
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/IRecordWriter.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.file;
+
+import java.io.File;
+
+public interface IRecordWriter {
+
+ public void close();
+
+ public void write(Object[] record) throws Exception;
+
+
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/LineFileScanOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/LineFileScanOperatorDescriptor.java
new file mode 100644
index 0000000..c2e5a3b
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/LineFileScanOperatorDescriptor.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.file;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class LineFileScanOperatorDescriptor extends
+ AbstractFileScanOperatorDescriptor {
+ private static class LineReaderImpl extends RecordReader {
+ private File file;
+
+ LineReaderImpl(File file) throws Exception {
+ super(new Object[]{file});
+ this.file = file;
+ }
+
+ @Override
+ public InputStream createInputStream(Object[] args) throws Exception{
+ this.file = (File)args[0];
+ return new FileInputStream(file) ;
+ }
+ }
+
+ private static final long serialVersionUID = 1L;
+
+ public LineFileScanOperatorDescriptor(JobSpecification spec,
+ FileSplit[] splits, RecordDescriptor recordDescriptor) {
+ super(spec, splits, recordDescriptor);
+ }
+
+ @Override
+ protected IRecordReader createRecordReader(File file, RecordDescriptor desc)
+ throws Exception {
+ return new LineReaderImpl(file);
+ }
+
+ @Override
+ protected void configure() throws Exception {
+ // currently a no-op, but is meant to initialize , if required before it is asked
+ // to create a record reader
+ // this is executed at the node and is useful for operators that could not be
+ // initialized from the client completely, because of lack of information specific
+ // to the node where the operator gets executed.
+
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/LineFileWriteOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/LineFileWriteOperatorDescriptor.java
new file mode 100644
index 0000000..5e7bfef
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/LineFileWriteOperatorDescriptor.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.file;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class LineFileWriteOperatorDescriptor extends AbstractFileWriteOperatorDescriptor {
+
+ private static class LineWriterImpl extends RecordWriter {
+
+ File file;
+
+ LineWriterImpl(File file,int []columns, char separator) throws Exception {
+ super(columns,separator, new Object[]{file});
+ this.file=file;
+ }
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public OutputStream createOutputStream(Object[] args) throws Exception {
+ return new FileOutputStream((File)args[0]);
+ }
+ }
+
+ private int[]columns ;
+ private char separator;
+
+
+ public LineFileWriteOperatorDescriptor(JobSpecification spec,
+ FileSplit[] splits) {
+ this(spec, splits, null, RecordWriter.COMMA);
+ }
+
+ public LineFileWriteOperatorDescriptor(JobSpecification spec,
+ FileSplit[] splits, int[] columns) {
+ this(spec, splits, columns, RecordWriter.COMMA);
+ }
+
+ public LineFileWriteOperatorDescriptor(JobSpecification spec,
+ FileSplit[] splits, int[] columns, char separator) {
+ super(spec, splits);
+ this.columns = columns;
+ this.separator = separator;
+ }
+
+
+ @Override
+ protected IRecordWriter createRecordWriter(File file,int index) throws Exception {
+ return new LineWriterImpl(file,columns,separator);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/RecordFileScanOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/RecordFileScanOperatorDescriptor.java
new file mode 100644
index 0000000..9df04f9
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/RecordFileScanOperatorDescriptor.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.file;
+
+import java.io.BufferedInputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class RecordFileScanOperatorDescriptor extends AbstractFileScanOperatorDescriptor {
+ private static final long serialVersionUID = 1L;
+
+ public RecordFileScanOperatorDescriptor(JobSpecification spec, FileSplit[] splits, RecordDescriptor recordDescriptor) {
+ super(spec, splits, recordDescriptor);
+ }
+
+ private static class RecordReaderImpl implements IRecordReader {
+ private RecordDescriptor recordDesc;
+ private DataInputStream in;
+
+ RecordReaderImpl(File file, RecordDescriptor recordDesc) throws Exception {
+ this.in = new DataInputStream(new BufferedInputStream(new FileInputStream(file)));
+ this.recordDesc = recordDesc;
+ }
+
+ @Override
+ public boolean read(Object[] record) throws Exception {
+ in.mark(1);
+ if (in.read() < 0) {
+ return false;
+ }
+ in.reset();
+ for (int i = 0; i < record.length; ++i) {
+ record[i] = recordDesc.getFields()[i].deserialize(in);
+ }
+ return true;
+ }
+
+ @Override
+ public void close() {
+ try {
+ in.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ @Override
+ protected IRecordReader createRecordReader(File file, RecordDescriptor desc) throws Exception {
+ return new RecordReaderImpl(file, desc);
+ }
+
+ @Override
+ protected void configure() throws Exception {
+ // currently a no-op, but is meant to initialize , if required before it is asked
+ // to create a record reader
+ // this is executed at the node and is useful for operators that could not be
+ // initialized from the client completely, because of lack of information specific
+ // to the node where the operator gets executed.
+
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/RecordReader.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/RecordReader.java
new file mode 100644
index 0000000..63f8f7e
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/RecordReader.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.file;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+public abstract class RecordReader implements IRecordReader {
+
+ private final BufferedReader bufferedReader;
+ private InputStream inputStream;
+
+ @Override
+ public void close() {
+ try {
+ bufferedReader.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public boolean read(Object[] record) throws Exception {
+ String line = bufferedReader.readLine();
+ if (line == null) {
+ return false;
+ }
+ record[0] = line;
+ return true;
+ }
+
+ public abstract InputStream createInputStream(Object[] args) throws Exception;
+
+ public RecordReader(Object[] args) throws Exception{
+ try{
+ bufferedReader = new BufferedReader(new InputStreamReader(createInputStream(args)));
+ }catch(Exception e){
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/RecordWriter.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/RecordWriter.java
new file mode 100644
index 0000000..f92686d
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/file/RecordWriter.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.file;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+public abstract class RecordWriter implements IRecordWriter{
+
+
+ protected final BufferedWriter bufferedWriter;
+ protected final int[] columns;
+ protected final char separator;
+
+ public static final char COMMA = ',';
+
+ public RecordWriter(Object [] args) throws Exception{
+ OutputStream outputStream = createOutputStream(args);
+ if(outputStream != null){
+ bufferedWriter = new BufferedWriter(new OutputStreamWriter(createOutputStream(args)));
+ }else{
+ bufferedWriter = null;
+ }
+ this.columns = null;
+ this.separator = COMMA;
+ }
+
+ public RecordWriter(int []columns, char separator, Object[] args) throws Exception{
+ OutputStream outputStream = createOutputStream(args);
+ if(outputStream != null){
+ bufferedWriter = new BufferedWriter(new OutputStreamWriter(outputStream));
+ }else{
+ bufferedWriter = null;
+ }
+ this.columns = columns;
+ this.separator = separator;
+ }
+
+ @Override
+ public void close() {
+ try {
+ bufferedWriter.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public void write(Object[] record) throws Exception {
+ if (columns == null) {
+ for (int i = 0; i < record.length; ++i) {
+ if (i != 0) {
+ bufferedWriter.write(separator);
+ }
+ bufferedWriter.write(String.valueOf(record[i]));
+ }
+ } else {
+ for (int i = 0; i < columns.length; ++i) {
+ if (i != 0) {
+ bufferedWriter.write(separator);
+ }
+ bufferedWriter.write(String.valueOf(record[columns[i]]));
+ }
+ }
+ bufferedWriter.write("\n");
+ }
+
+ public abstract OutputStream createOutputStream(Object[] args) throws Exception;
+
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/GroupingHashTable.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/GroupingHashTable.java
new file mode 100644
index 0000000..86c247d
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/GroupingHashTable.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.group;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.comm.io.FrameTuplePairComparator;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+class GroupingHashTable {
+ /**
+ * The pointers in the link store 3 int values for each entry in the
+ * hashtable: (bufferIdx, tIndex, accumulatorIdx).
+ *
+ * @author vinayakb
+ */
+ private static class Link {
+ private static final int INIT_POINTERS_SIZE = 9;
+
+ int[] pointers;
+ int size;
+
+ Link() {
+ pointers = new int[INIT_POINTERS_SIZE];
+ size = 0;
+ }
+
+ void add(int bufferIdx, int tIndex, int accumulatorIdx) {
+ while (size + 3 > pointers.length) {
+ pointers = Arrays.copyOf(pointers, pointers.length * 2);
+ }
+ pointers[size++] = bufferIdx;
+ pointers[size++] = tIndex;
+ pointers[size++] = accumulatorIdx;
+ }
+ }
+
+ private static final int INIT_ACCUMULATORS_SIZE = 8;
+ private final HyracksContext ctx;
+ private final FrameTupleAppender appender;
+ private final List<ByteBuffer> buffers;
+ private final Link[] table;
+ private IAccumulatingAggregator[] accumulators;
+ private int accumulatorSize;
+
+ private int lastBIndex;
+ private final int[] fields;
+ private final int[] storedKeys;
+ private final IBinaryComparator[] comparators;
+ private final FrameTuplePairComparator ftpc;
+ private final ITuplePartitionComputer tpc;
+ private final IAccumulatingAggregatorFactory aggregatorFactory;
+ private final RecordDescriptor inRecordDescriptor;
+ private final RecordDescriptor outRecordDescriptor;
+
+ private final FrameTupleAccessor storedKeysAccessor;
+
+ GroupingHashTable(HyracksContext ctx, int[] fields, IBinaryComparatorFactory[] comparatorFactories,
+ ITuplePartitionComputerFactory tpcf, IAccumulatingAggregatorFactory aggregatorFactory,
+ RecordDescriptor inRecordDescriptor, RecordDescriptor outRecordDescriptor, int tableSize) {
+ this.ctx = ctx;
+ appender = new FrameTupleAppender(ctx);
+ buffers = new ArrayList<ByteBuffer>();
+ table = new Link[tableSize];
+ accumulators = new IAccumulatingAggregator[INIT_ACCUMULATORS_SIZE];
+ accumulatorSize = 0;
+ this.fields = fields;
+ storedKeys = new int[fields.length];
+ ISerializerDeserializer[] storedKeySerDeser = new ISerializerDeserializer[fields.length];
+ for (int i = 0; i < fields.length; ++i) {
+ storedKeys[i] = i;
+ storedKeySerDeser[i] = inRecordDescriptor.getFields()[fields[i]];
+ }
+ comparators = new IBinaryComparator[comparatorFactories.length];
+ for (int i = 0; i < comparatorFactories.length; ++i) {
+ comparators[i] = comparatorFactories[i].createBinaryComparator();
+ }
+ ftpc = new FrameTuplePairComparator(fields, storedKeys, comparators);
+ tpc = tpcf.createPartitioner();
+ this.aggregatorFactory = aggregatorFactory;
+ this.inRecordDescriptor = inRecordDescriptor;
+ this.outRecordDescriptor = outRecordDescriptor;
+ RecordDescriptor storedKeysRecordDescriptor = new RecordDescriptor(storedKeySerDeser);
+ storedKeysAccessor = new FrameTupleAccessor(ctx, storedKeysRecordDescriptor);
+ lastBIndex = -1;
+ addNewBuffer();
+ }
+
+ private void addNewBuffer() {
+ ByteBuffer buffer = ctx.getResourceManager().allocateFrame();
+ buffer.position(0);
+ buffer.limit(buffer.capacity());
+ buffers.add(buffer);
+ appender.reset(buffer, true);
+ ++lastBIndex;
+ }
+
+ private void flushFrame(FrameTupleAppender appender, IFrameWriter writer) throws HyracksDataException {
+ ByteBuffer frame = appender.getBuffer();
+ frame.position(0);
+ frame.limit(frame.capacity());
+ writer.nextFrame(appender.getBuffer());
+ appender.reset(appender.getBuffer(), true);
+ }
+
+ void insert(FrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
+ int entry = tpc.partition(accessor, tIndex, table.length);
+ Link link = table[entry];
+ if (link == null) {
+ link = table[entry] = new Link();
+ }
+ IAccumulatingAggregator aggregator = null;
+ for (int i = 0; i < link.size; i += 3) {
+ int sbIndex = link.pointers[i];
+ int stIndex = link.pointers[i + 1];
+ int saIndex = link.pointers[i + 2];
+ storedKeysAccessor.reset(buffers.get(sbIndex));
+ int c = ftpc.compare(accessor, tIndex, storedKeysAccessor, stIndex);
+ if (c == 0) {
+ aggregator = accumulators[saIndex];
+ break;
+ }
+ }
+ if (aggregator == null) {
+ // Did not find the key. Insert a new entry.
+ if (!appender.appendProjection(accessor, tIndex, fields)) {
+ addNewBuffer();
+ if (!appender.appendProjection(accessor, tIndex, fields)) {
+ throw new IllegalStateException();
+ }
+ }
+ int sbIndex = lastBIndex;
+ int stIndex = appender.getTupleCount() - 1;
+ if (accumulatorSize >= accumulators.length) {
+ accumulators = Arrays.copyOf(accumulators, accumulators.length * 2);
+ }
+ int saIndex = accumulatorSize++;
+ aggregator = accumulators[saIndex] = aggregatorFactory.createAggregator(inRecordDescriptor,
+ outRecordDescriptor);
+ aggregator.init(accessor, tIndex);
+ link.add(sbIndex, stIndex, saIndex);
+ }
+ aggregator.accumulate(accessor, tIndex);
+ }
+
+ void write(IFrameWriter writer) throws HyracksDataException {
+ ByteBuffer buffer = ctx.getResourceManager().allocateFrame();
+ appender.reset(buffer, true);
+ for (int i = 0; i < table.length; ++i) {
+ Link link = table[i];
+ if (link != null) {
+ for (int j = 0; j < link.size; j += 3) {
+ int bIndex = link.pointers[j];
+ int tIndex = link.pointers[j + 1];
+ int aIndex = link.pointers[j + 2];
+ ByteBuffer keyBuffer = buffers.get(bIndex);
+ storedKeysAccessor.reset(keyBuffer);
+ IAccumulatingAggregator aggregator = accumulators[aIndex];
+ while (!aggregator.output(appender, storedKeysAccessor, tIndex)) {
+ flushFrame(appender, writer);
+ }
+ }
+ }
+ }
+ if (appender.getTupleCount() != 0) {
+ flushFrame(appender, writer);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/HashGroupOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/HashGroupOperatorDescriptor.java
new file mode 100644
index 0000000..2fc6b63
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/HashGroupOperatorDescriptor.java
@@ -0,0 +1,192 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.group;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePullable;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.AbstractActivityNode;
+import edu.uci.ics.hyracks.coreops.base.AbstractOperatorDescriptor;
+
+public class HashGroupOperatorDescriptor extends AbstractOperatorDescriptor {
+ private static final String HASHTABLE = "hashtable";
+
+ private static final long serialVersionUID = 1L;
+
+ private final int[] keys;
+ private final ITuplePartitionComputerFactory tpcf;
+ private final IBinaryComparatorFactory[] comparatorFactories;
+ private final IAccumulatingAggregatorFactory aggregatorFactory;
+ private final int tableSize;
+
+ public HashGroupOperatorDescriptor(JobSpecification spec, int[] keys, ITuplePartitionComputerFactory tpcf,
+ IBinaryComparatorFactory[] comparatorFactories, IAccumulatingAggregatorFactory aggregatorFactory,
+ RecordDescriptor recordDescriptor, int tableSize) {
+ super(spec, 1, 1);
+ this.keys = keys;
+ this.tpcf = tpcf;
+ this.comparatorFactories = comparatorFactories;
+ this.aggregatorFactory = aggregatorFactory;
+ recordDescriptors[0] = recordDescriptor;
+ this.tableSize = tableSize;
+ }
+
+ @Override
+ public void contributeTaskGraph(IActivityGraphBuilder builder) {
+ HashBuildActivity ha = new HashBuildActivity();
+ builder.addTask(ha);
+
+ OutputActivity oa = new OutputActivity();
+ builder.addTask(oa);
+
+ builder.addSourceEdge(0, ha, 0);
+ builder.addTargetEdge(0, oa, 0);
+ builder.addBlockingEdge(ha, oa);
+ }
+
+ private class HashBuildActivity extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final HyracksContext ctx, JobPlan plan,
+ final IOperatorEnvironment env, int partition) {
+ final RecordDescriptor rd0 = plan.getJobSpecification()
+ .getOperatorInputRecordDescriptor(getOperatorId(), 0);
+ final FrameTupleAccessor accessor = new FrameTupleAccessor(ctx, rd0);
+ return new IOperatorNodePushable() {
+ private GroupingHashTable table;
+
+ @Override
+ public void open() throws HyracksDataException {
+ table = new GroupingHashTable(ctx, keys, comparatorFactories, tpcf, aggregatorFactory, rd0,
+ recordDescriptors[0], tableSize);
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ accessor.reset(buffer);
+ int tupleCount = accessor.getTupleCount();
+ for (int i = 0; i < tupleCount; ++i) {
+ table.insert(accessor, i);
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ env.set(HASHTABLE, table);
+ }
+
+ @Override
+ public void setFrameWriter(int index, IFrameWriter writer) {
+ throw new IllegalArgumentException();
+ }
+ };
+ }
+
+ @Override
+ public IOperatorDescriptor getOwner() {
+ return HashGroupOperatorDescriptor.this;
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+ }
+
+ private class OutputActivity extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(HyracksContext ctx, JobPlan plan, final IOperatorEnvironment env,
+ int partition) {
+ return new IOperatorNodePushable() {
+ private IFrameWriter writer;
+
+ @Override
+ public void open() throws HyracksDataException {
+ GroupingHashTable table = (GroupingHashTable) env.get(HASHTABLE);
+ writer.open();
+ table.write(writer);
+ writer.close();
+ env.set(HASHTABLE, null);
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ // do nothing
+ }
+
+ @Override
+ public void setFrameWriter(int index, IFrameWriter writer) {
+ if (index != 0) {
+ throw new IllegalArgumentException();
+ }
+ this.writer = writer;
+ }
+ };
+ }
+
+ @Override
+ public IOperatorDescriptor getOwner() {
+ return HashGroupOperatorDescriptor.this;
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/IAccumulatingAggregator.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/IAccumulatingAggregator.java
new file mode 100644
index 0000000..d9e9337
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/IAccumulatingAggregator.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.group;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAppender;
+
+public interface IAccumulatingAggregator {
+ /**
+ * Called once per aggregator before calling accumulate for the first time.
+ *
+ * @param accessor
+ * - Accessor to the data tuple.
+ * @param tIndex
+ * - Index of the tuple in the accessor.
+ * @throws HyracksDataException
+ */
+ public void init(FrameTupleAccessor accessor, int tIndex) throws HyracksDataException;
+
+ /**
+ * Called once per tuple that belongs to this group.
+ *
+ * @param accessor
+ * - Accessor to data tuple.
+ * @param tIndex
+ * - Index of tuple in the accessor.
+ * @throws HyracksDataException
+ */
+ public void accumulate(FrameTupleAccessor accessor, int tIndex) throws HyracksDataException;
+
+ /**
+ * Called finally to emit output. This method is called until it returns true. The method is free to
+ * write out output to the provided appender until there is no more space and return false. It is the
+ * caller's responsibility to flush and make room in the appender before this method is called again.
+ *
+ * @param appender
+ * - Appender to write output to.
+ * @param accessor
+ * - Accessor to access the key.
+ * @param tIndex
+ * - Tuple index of the key in the accessor.
+ * @return true if all output is written, false if the appender is full.
+ * @throws HyracksDataException
+ */
+ public boolean output(FrameTupleAppender appender, FrameTupleAccessor accessor, int tIndex)
+ throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/IAccumulatingAggregatorFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/IAccumulatingAggregatorFactory.java
new file mode 100644
index 0000000..e5c935c
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/IAccumulatingAggregatorFactory.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.group;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public interface IAccumulatingAggregatorFactory extends Serializable {
+ IAccumulatingAggregator createAggregator(RecordDescriptor inRecordDesc, RecordDescriptor outRecordDescriptor);
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/IGroupAggregator.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/IGroupAggregator.java
new file mode 100644
index 0000000..69a0f47
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/IGroupAggregator.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.group;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.dataflow.IDataReader;
+import edu.uci.ics.hyracks.api.dataflow.IDataWriter;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface IGroupAggregator extends Serializable {
+ public void aggregate(IDataReader<Object[]> reader, IDataWriter<Object[]> writer) throws HyracksDataException;
+
+ public void close() throws Exception;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/PreclusteredGroupOperator.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/PreclusteredGroupOperator.java
new file mode 100644
index 0000000..4af62dc
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/PreclusteredGroupOperator.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.group;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataReader;
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataWriter;
+import edu.uci.ics.hyracks.api.dataflow.value.IComparator;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.coreops.base.IOpenableDataWriterOperator;
+
+public class PreclusteredGroupOperator implements IOpenableDataWriterOperator {
+ private final int[] groupFields;
+
+ private final IComparator[] comparators;
+
+ private final IGroupAggregator aggregator;
+
+ private Object[] lastData;
+
+ private IOpenableDataWriter<Object[]> writer;
+
+ private List<Object[]> buffer;
+
+ private IOpenableDataReader<Object[]> reader;
+
+ public PreclusteredGroupOperator(int[] groupFields, IComparator[] comparators, IGroupAggregator aggregator) {
+ this.groupFields = groupFields;
+ this.comparators = comparators;
+ this.aggregator = aggregator;
+ buffer = new ArrayList<Object[]>();
+ reader = new IOpenableDataReader<Object[]>() {
+ private int idx;
+
+ @Override
+ public void open() {
+ idx = 0;
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public Object[] readData() {
+ return idx >= buffer.size() ? null : buffer.get(idx++);
+ }
+ };
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ if (!buffer.isEmpty()) {
+ aggregate();
+ }
+ writer.close();
+ }
+
+ private void aggregate() throws HyracksDataException {
+ reader.open();
+ aggregator.aggregate(reader, writer);
+ reader.close();
+ buffer.clear();
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ lastData = null;
+ writer.open();
+ }
+
+ @Override
+ public void setDataWriter(int index, IOpenableDataWriter<Object[]> writer) {
+ if (index != 0) {
+ throw new IllegalArgumentException();
+ }
+ this.writer = writer;
+ }
+
+ @Override
+ public void writeData(Object[] data) throws HyracksDataException {
+ if (lastData != null && compare(data, lastData) != 0) {
+ aggregate();
+ }
+ lastData = data;
+ buffer.add(data);
+ }
+
+ private int compare(Object[] d1, Object[] d2) {
+ for (int i = 0; i < groupFields.length; ++i) {
+ int fIdx = groupFields[i];
+ int c = comparators[i].compare(d1[fIdx], d2[fIdx]);
+ if (c != 0) {
+ return c;
+ }
+ }
+ return 0;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/PreclusteredGroupOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/PreclusteredGroupOperatorDescriptor.java
new file mode 100644
index 0000000..0232d73
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/group/PreclusteredGroupOperatorDescriptor.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.group;
+
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePullable;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.AbstractSingleActivityOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.util.DeserializedOperatorNodePushable;
+
+public class PreclusteredGroupOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+ private final int[] groupFields;
+ private final IGroupAggregator aggregator;
+ private final IComparatorFactory[] comparatorFactories;
+
+ private static final long serialVersionUID = 1L;
+
+ public PreclusteredGroupOperatorDescriptor(JobSpecification spec, int[] groupFields,
+ IComparatorFactory[] comparatorFactories, IGroupAggregator aggregator, RecordDescriptor recordDescriptor) {
+ super(spec, 1, 1);
+ this.groupFields = groupFields;
+ this.comparatorFactories = comparatorFactories;
+ this.aggregator = aggregator;
+ recordDescriptors[0] = recordDescriptor;
+ }
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ IComparator[] comparators = new IComparator[comparatorFactories.length];
+ for (int i = 0; i < comparatorFactories.length; ++i) {
+ comparators[i] = comparatorFactories[i].createComparator();
+ }
+ return new DeserializedOperatorNodePushable(ctx, new PreclusteredGroupOperator(groupFields, comparators,
+ aggregator), plan, getActivityNodeId());
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/AbstractHadoopFileScanOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/AbstractHadoopFileScanOperatorDescriptor.java
new file mode 100644
index 0000000..5ee84e4
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/AbstractHadoopFileScanOperatorDescriptor.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.hadoop;
+
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.Counters.Counter;
+
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataWriter;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePullable;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.AbstractSingleActivityOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.base.IOpenableDataWriterOperator;
+import edu.uci.ics.hyracks.coreops.file.IRecordReader;
+import edu.uci.ics.hyracks.coreops.util.DeserializedOperatorNodePushable;
+import edu.uci.ics.hyracks.hadoop.util.HadoopFileSplit;
+
+public abstract class AbstractHadoopFileScanOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+ private static final long serialVersionUID = 1L;
+ protected HadoopFileSplit[] splits;
+
+ public AbstractHadoopFileScanOperatorDescriptor(JobSpecification spec, HadoopFileSplit[] splits,
+ RecordDescriptor recordDescriptor) {
+ super(spec, 0, 1);
+ recordDescriptors[0] = recordDescriptor;
+ this.splits = splits;
+ }
+
+ protected abstract IRecordReader createRecordReader(HadoopFileSplit fileSplit, RecordDescriptor desc)
+ throws Exception;
+
+ protected class FileScanOperator implements IOpenableDataWriterOperator {
+ private IOpenableDataWriter<Object[]> writer;
+ private int index;
+
+ FileScanOperator(int index) {
+ this.index = index;
+ }
+
+ @Override
+ public void setDataWriter(int index, IOpenableDataWriter<Object[]> writer) {
+ if (index != 0) {
+ throw new IndexOutOfBoundsException("Invalid index: " + index);
+ }
+ this.writer = writer;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ HadoopFileSplit split = splits[index];
+ RecordDescriptor desc = recordDescriptors[0];
+ try {
+ IRecordReader reader = createRecordReader(split, desc);
+ if (desc == null) {
+ desc = recordDescriptors[0];
+ }
+ writer.open();
+ try {
+ while (true) {
+ Object[] record = new Object[desc.getFields().length];
+ if (!reader.read(record)) {
+ break;
+ }
+ writer.writeData(record);
+ }
+ } finally {
+ reader.close();
+ writer.close();
+ }
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ // do nothing
+ }
+
+ @Override
+ public void writeData(Object[] data) throws HyracksDataException {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ protected Reporter createReporter() {
+ return new Reporter() {
+ @Override
+ public Counter getCounter(Enum<?> name) {
+ return null;
+ }
+
+ @Override
+ public Counter getCounter(String group, String name) {
+ return null;
+ }
+
+ @Override
+ public InputSplit getInputSplit() throws UnsupportedOperationException {
+ return null;
+ }
+
+ @Override
+ public void incrCounter(Enum<?> key, long amount) {
+
+ }
+
+ @Override
+ public void incrCounter(String group, String counter, long amount) {
+
+ }
+
+ @Override
+ public void progress() {
+
+ }
+
+ @Override
+ public void setStatus(String status) {
+
+ }
+ };
+ }
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ return new DeserializedOperatorNodePushable(ctx, new FileScanOperator(partition), plan, getActivityNodeId());
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/AbstractHadoopOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/AbstractHadoopOperatorDescriptor.java
new file mode 100644
index 0000000..4724d80
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/AbstractHadoopOperatorDescriptor.java
@@ -0,0 +1,172 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.hadoop;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.Counters.Counter;
+
+import edu.uci.ics.dcache.client.DCacheClient;
+import edu.uci.ics.hyracks.api.dataflow.IDataWriter;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.coreops.base.AbstractSingleActivityOperatorDescriptor;
+import edu.uci.ics.hyracks.hadoop.util.DatatypeHelper;
+import edu.uci.ics.hyracks.hadoop.util.IHadoopClassFactory;
+
+public abstract class AbstractHadoopOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+
+
+ protected static class DataWritingOutputCollector<K, V> implements OutputCollector<K, V> {
+ private IDataWriter<Object[]> writer;
+
+ public DataWritingOutputCollector() {
+ }
+
+ public DataWritingOutputCollector(IDataWriter<Object[]> writer) {
+ this.writer = writer;
+ }
+
+ @Override
+ public void collect(Object key, Object value) throws IOException {
+ writer.writeData(new Object[] { key, value });
+ }
+
+ public void setWriter(IDataWriter<Object[]> writer) {
+ this.writer = writer;
+ }
+ }
+
+ public static String MAPRED_CACHE_FILES = "mapred.cache.files";
+ public static String MAPRED_CACHE_LOCALFILES = "mapred.cache.localFiles";
+
+ private static final long serialVersionUID = 1L;
+ private final HashMap<String, String> jobConfMap;
+ private IHadoopClassFactory hadoopClassFactory;
+
+ public abstract RecordDescriptor getRecordDescriptor(JobConf jobConf);
+
+ public AbstractHadoopOperatorDescriptor(JobSpecification spec, RecordDescriptor recordDescriptor, JobConf jobConf,
+ IHadoopClassFactory hadoopOperatorFactory) {
+ super(spec, 1, 1);
+ jobConfMap = DatatypeHelper.jobConf2HashMap(jobConf);
+ this.hadoopClassFactory = hadoopOperatorFactory;
+ if(recordDescriptor != null){
+ recordDescriptors[0]= recordDescriptor;
+ }else{
+ recordDescriptors[0] = getRecordDescriptor(jobConf);
+ }
+ }
+
+
+ public HashMap<String, String> getJobConfMap() {
+ return jobConfMap;
+ }
+
+
+ public IHadoopClassFactory getHadoopClassFactory() {
+ return hadoopClassFactory;
+ }
+
+
+ public void setHadoopClassFactory(IHadoopClassFactory hadoopClassFactory) {
+ this.hadoopClassFactory = hadoopClassFactory;
+ }
+
+
+ protected Reporter createReporter() {
+ return new Reporter() {
+ @Override
+ public Counter getCounter(Enum<?> name) {
+ return null;
+ }
+
+ @Override
+ public Counter getCounter(String group, String name) {
+ return null;
+ }
+
+
+ @Override
+ public InputSplit getInputSplit() throws UnsupportedOperationException {
+ return null;
+ }
+
+ @Override
+ public void incrCounter(Enum<?> key, long amount) {
+
+ }
+
+ @Override
+ public void incrCounter(String group, String counter, long amount) {
+
+ }
+
+ @Override
+ public void progress() {
+
+ }
+
+ @Override
+ public void setStatus(String status) {
+
+ }
+ };
+ }
+
+ public JobConf getJobConf() {
+ return DatatypeHelper.hashMap2JobConf(jobConfMap);
+ }
+
+ public void populateCache(JobConf jobConf) {
+ String cache = jobConf.get(MAPRED_CACHE_FILES);
+ System.out.println("cache:" + cache);
+ if (cache == null) {
+ return;
+ }
+ String localCache = jobConf.get(MAPRED_CACHE_LOCALFILES);
+ System.out.println("localCache:" + localCache);
+ if (localCache != null) {
+ return;
+ }
+ localCache = "";
+ StringTokenizer cacheTokenizer = new StringTokenizer(cache, ",");
+ while (cacheTokenizer.hasMoreTokens()) {
+ if (!"".equals(localCache)) {
+ localCache += ",";
+ }
+ try {
+ localCache += DCacheClient.get().get(cacheTokenizer.nextToken());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ jobConf.set(MAPRED_CACHE_LOCALFILES, localCache);
+ System.out.println("localCache:" + localCache);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/HDFSWriteOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/HDFSWriteOperatorDescriptor.java
new file mode 100644
index 0000000..f2487f9
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/HDFSWriteOperatorDescriptor.java
@@ -0,0 +1,176 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.hadoop;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.SequenceFile.Writer;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.hadoop.mapred.JobConf;
+
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.coreops.file.AbstractFileWriteOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.file.FileSplit;
+import edu.uci.ics.hyracks.coreops.file.IRecordWriter;
+import edu.uci.ics.hyracks.coreops.file.RecordWriter;
+import edu.uci.ics.hyracks.hadoop.util.DatatypeHelper;
+
+
+
+public class HDFSWriteOperatorDescriptor extends
+ AbstractFileWriteOperatorDescriptor {
+
+ private static String nullWritableClassName = NullWritable.class.getName();
+
+ private static class HDFSLineWriterImpl extends RecordWriter {
+
+ HDFSLineWriterImpl(FileSystem fileSystem, String hdfsPath, int[] columns, char separator)
+ throws Exception {
+ super(columns,separator,new Object[]{fileSystem,hdfsPath});
+ }
+
+ @Override
+ public OutputStream createOutputStream(Object[] args) throws Exception {
+ FSDataOutputStream fs = ((FileSystem)args[0]).create(new Path((String)args[1]));
+ return fs;
+ }
+
+ @Override
+ public void write(Object[] record) throws Exception {
+ if(!nullWritableClassName.equals((record[0].getClass().getName()))){
+ bufferedWriter.write(String.valueOf(record[0]));
+ }
+ if(!nullWritableClassName.equals((record[1].getClass().getName()))){
+ bufferedWriter.write(separator);
+ bufferedWriter.write(String.valueOf(record[1]));
+ }
+ bufferedWriter.write("\n");
+ }
+ }
+
+ private static class HDFSSequenceWriterImpl extends RecordWriter {
+
+ private Writer writer;
+
+ HDFSSequenceWriterImpl(FileSystem fileSystem, String hdfsPath, Writer writer)
+ throws Exception {
+ super(null,COMMA,new Object[]{fileSystem,hdfsPath});
+ this.writer = writer;
+ }
+
+ @Override
+ public OutputStream createOutputStream(Object[] args) throws Exception {
+ return null;
+ }
+
+ @Override
+ public void close() {
+ try {
+ writer.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public void write(Object[] record) throws Exception {
+ Object key = record[0];
+ Object value = record[1];
+ writer.append(key, value);
+ }
+
+ }
+
+ private static final long serialVersionUID = 1L;
+ private static final char COMMA = ',';
+ private char separator;
+ private boolean sequenceFileOutput = false;
+ private String keyClassName;
+ private String valueClassName;
+ Map<String,String> jobConfMap;
+
+
+ @Override
+ protected IRecordWriter createRecordWriter(File file,int index) throws Exception {
+ JobConf conf = DatatypeHelper.hashMap2JobConf((HashMap)jobConfMap);
+ System.out.println("replication:" + conf.get("dfs.replication"));
+ FileSystem fileSystem = null;
+ try{
+ fileSystem = FileSystem.get(conf);
+ }catch(IOException ioe){
+ ioe.printStackTrace();
+ }
+ Path path = new Path(file.getAbsolutePath());
+ checkIfCanWriteToHDFS(new FileSplit[]{new FileSplit("localhost",file)});
+ FSDataOutputStream outputStream = fileSystem.create(path);
+ outputStream.close();
+ if(sequenceFileOutput){
+ Class keyClass = Class.forName(keyClassName);
+ Class valueClass= Class.forName(valueClassName);
+ conf.setClass("mapred.output.compression.codec", GzipCodec.class, CompressionCodec.class);
+ Writer writer = SequenceFile.createWriter(fileSystem, conf,path, keyClass, valueClass);
+ return new HDFSSequenceWriterImpl(fileSystem, file.getAbsolutePath(), writer);
+ }else{
+ return new HDFSLineWriterImpl(fileSystem, file.getAbsolutePath(), null, COMMA);
+ }
+ }
+
+ private boolean checkIfCanWriteToHDFS(FileSplit[] fileSplits) throws Exception{
+ boolean canWrite = true;
+ JobConf conf = DatatypeHelper.hashMap2JobConf((HashMap)jobConfMap);
+ FileSystem fileSystem = null;
+ try{
+ fileSystem = FileSystem.get(conf);
+ for(FileSplit fileSplit : fileSplits){
+ Path path = new Path(fileSplit.getLocalFile().getAbsolutePath());
+ canWrite = !fileSystem.exists(path);
+ if(!canWrite){
+ throw new Exception(" Output path : already exists");
+ }
+ }
+ }catch(IOException ioe){
+ ioe.printStackTrace();
+ throw ioe;
+ }
+ return canWrite;
+ }
+
+
+ public HDFSWriteOperatorDescriptor(JobSpecification jobSpec,Map<String,String> jobConfMap, FileSplit[] fileSplits,char seperator,boolean sequenceFileOutput,String keyClassName, String valueClassName) throws Exception{
+ super(jobSpec,fileSplits);
+ this.jobConfMap = jobConfMap;
+ checkIfCanWriteToHDFS(fileSplits);
+ this.separator = seperator;
+ this.sequenceFileOutput = sequenceFileOutput;
+ this.keyClassName = keyClassName;
+ this.valueClassName = valueClassName;
+ }
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/HadoopMapperOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/HadoopMapperOperatorDescriptor.java
new file mode 100644
index 0000000..5385e44
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/HadoopMapperOperatorDescriptor.java
@@ -0,0 +1,157 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.hadoop;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataWriter;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePullable;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.IOpenableDataWriterOperator;
+import edu.uci.ics.hyracks.coreops.util.DeserializedOperatorNodePushable;
+import edu.uci.ics.hyracks.hadoop.util.ClasspathBasedHadoopClassFactory;
+import edu.uci.ics.hyracks.hadoop.util.DatatypeHelper;
+import edu.uci.ics.hyracks.hadoop.util.IHadoopClassFactory;
+
+public class HadoopMapperOperatorDescriptor<K1, V1, K2, V2> extends AbstractHadoopOperatorDescriptor {
+ private class MapperOperator implements IOpenableDataWriterOperator {
+ private OutputCollector<K2, V2> output;
+ private Reporter reporter;
+ private Mapper<K1, V1, K2, V2> mapper;
+ private IOpenableDataWriter<Object[]> writer;
+
+ @Override
+ public void close() throws HyracksDataException {
+ try {
+ mapper.close();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ writer.close();
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ JobConf jobConf = getJobConf();
+ populateCache(jobConf);
+ try {
+ mapper = createMapper();
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ // -- - configure - --
+ mapper.configure(jobConf);
+ writer.open();
+ output = new DataWritingOutputCollector<K2, V2>(writer);
+ reporter = createReporter();
+ }
+
+ @Override
+ public void setDataWriter(int index, IOpenableDataWriter<Object[]> writer) {
+ if (index != 0) {
+ throw new IllegalArgumentException();
+ }
+ this.writer = writer;
+ }
+
+ @Override
+ public void writeData(Object[] data) throws HyracksDataException {
+ try {
+ mapper.map((K1) data[0], (V1) data[1], output, reporter);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+
+ private static final long serialVersionUID = 1L;
+ private static final String mapClassNameKey = "mapred.mapper.class";
+ private Class<? extends Mapper> mapperClass;
+
+ public HadoopMapperOperatorDescriptor(JobSpecification spec, Class<? extends Mapper> mapperClass,
+ RecordDescriptor recordDescriptor, JobConf jobConf) {
+ super(spec, recordDescriptor, jobConf, null);
+ this.mapperClass = mapperClass;
+ }
+
+ public HadoopMapperOperatorDescriptor(JobSpecification spec, JobConf jobConf, IHadoopClassFactory hadoopClassFactory) {
+ super(spec, null, jobConf, hadoopClassFactory);
+ }
+
+ public RecordDescriptor getRecordDescriptor(JobConf conf){
+ RecordDescriptor recordDescriptor = null;
+ String mapOutputKeyClassName = conf.getMapOutputKeyClass().getName();
+ String mapOutputValueClassName = conf.getMapOutputValueClass().getName();
+ try{
+ if(getHadoopClassFactory() == null){
+ recordDescriptor = DatatypeHelper.createKeyValueRecordDescriptor((Class < ? extends Writable>) Class.forName(mapOutputKeyClassName),(Class < ? extends Writable>) Class.forName(mapOutputValueClassName));
+ }else{
+ recordDescriptor = DatatypeHelper.createKeyValueRecordDescriptor((Class < ? extends Writable>) getHadoopClassFactory().loadClass(mapOutputKeyClassName),(Class < ? extends Writable>) getHadoopClassFactory().loadClass(mapOutputValueClassName));
+ }
+ }catch(Exception e){
+ e.printStackTrace();
+ }
+ return recordDescriptor;
+ }
+
+ private Mapper<K1, V1, K2, V2> createMapper() throws Exception {
+ if (mapperClass != null) {
+ return mapperClass.newInstance();
+ } else {
+ String mapperClassName = super.getJobConfMap().get(mapClassNameKey);
+ Object mapper = getHadoopClassFactory().createMapper(mapperClassName);
+ mapperClass = (Class<? extends Mapper>) mapper.getClass();
+ return (Mapper)mapper;
+ }
+ }
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ return new DeserializedOperatorNodePushable(ctx, new MapperOperator(), plan, getActivityNodeId());
+ }
+
+ public Class<? extends Mapper> getMapperClass() {
+ return mapperClass;
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/HadoopReadOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/HadoopReadOperatorDescriptor.java
new file mode 100644
index 0000000..c1b917d
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/HadoopReadOperatorDescriptor.java
@@ -0,0 +1,237 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.hadoop;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.NetworkInterface;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.SequenceFileRecordReader;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import edu.uci.ics.hyracks.api.constraints.AbsoluteLocationConstraint;
+import edu.uci.ics.hyracks.api.constraints.LocationConstraint;
+import edu.uci.ics.hyracks.api.constraints.PartitionConstraint;
+import edu.uci.ics.hyracks.api.controller.IClusterController;
+import edu.uci.ics.hyracks.api.controller.INodeController;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.config.NCConfig;
+import edu.uci.ics.hyracks.coreops.file.IRecordReader;
+import edu.uci.ics.hyracks.hadoop.util.DatatypeHelper;
+import edu.uci.ics.hyracks.hadoop.util.HadoopAdapter;
+import edu.uci.ics.hyracks.hadoop.util.HadoopFileSplit;
+
+public class HadoopReadOperatorDescriptor extends AbstractHadoopFileScanOperatorDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ private String inputFormatClassName;
+ private Map<String, String> jobConfMap;
+
+ private static class HDFSCustomReader implements IRecordReader {
+ private RecordReader hadoopRecordReader;
+ private Class inputKeyClass;
+ private Class inputValueClass;
+ private Object key;
+ private Object value;
+
+ public HDFSCustomReader(Map<String, String> jobConfMap, HadoopFileSplit inputSplit,
+ String inputFormatClassName, Reporter reporter) {
+ try {
+ JobConf conf = DatatypeHelper.hashMap2JobConf((HashMap) jobConfMap);
+ FileSystem fileSystem = null;
+ try {
+ fileSystem = FileSystem.get(conf);
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ }
+
+ Class inputFormatClass = Class.forName(inputFormatClassName);
+ InputFormat inputFormat = (InputFormat) ReflectionUtils.newInstance(inputFormatClass, conf);
+ hadoopRecordReader = (RecordReader) inputFormat.getRecordReader(getFileSplit(inputSplit), conf,
+ reporter);
+ if (hadoopRecordReader instanceof SequenceFileRecordReader) {
+ inputKeyClass = ((SequenceFileRecordReader) hadoopRecordReader).getKeyClass();
+ inputValueClass = ((SequenceFileRecordReader) hadoopRecordReader).getValueClass();
+ } else {
+ inputKeyClass = hadoopRecordReader.createKey().getClass();
+ inputValueClass = hadoopRecordReader.createValue().getClass();
+ }
+ key = inputKeyClass.newInstance();
+ value = inputValueClass.newInstance();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ public Class getInputKeyClass() {
+ return inputKeyClass;
+ }
+
+ public void setInputKeyClass(Class inputKeyClass) {
+ this.inputKeyClass = inputKeyClass;
+ }
+
+ public Class getInputValueClass() {
+ return inputValueClass;
+ }
+
+ public void setInputValueClass(Class inputValueClass) {
+ this.inputValueClass = inputValueClass;
+ }
+
+ @Override
+ public void close() {
+ try {
+ hadoopRecordReader.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public boolean read(Object[] record) throws Exception {
+ if (!hadoopRecordReader.next(key, value)) {
+ return false;
+ }
+ if (record.length == 1) {
+ record[0] = value;
+ } else {
+ record[0] = key;
+ record[1] = value;
+ }
+ return true;
+ }
+
+ private FileSplit getFileSplit(HadoopFileSplit hadoopFileSplit) {
+ FileSplit fileSplit = new FileSplit(new Path(hadoopFileSplit.getFile()), hadoopFileSplit.getStart(),
+ hadoopFileSplit.getLength(), hadoopFileSplit.getHosts());
+ return fileSplit;
+ }
+ }
+
+ public HadoopReadOperatorDescriptor(Map<String, String> jobConfMap, JobSpecification spec,
+ HadoopFileSplit[] splits, String inputFormatClassName, RecordDescriptor recordDescriptor) {
+ super(spec, splits, recordDescriptor);
+ this.inputFormatClassName = inputFormatClassName;
+ this.jobConfMap = jobConfMap;
+ }
+
+ public HadoopReadOperatorDescriptor(Map<String, String> jobConfMap, InetSocketAddress nameNode,
+ JobSpecification spec, String inputFormatClassName, RecordDescriptor recordDescriptor) {
+ super(spec, null, recordDescriptor);
+ this.inputFormatClassName = inputFormatClassName;
+ this.jobConfMap = jobConfMap;
+ }
+
+ public HadoopReadOperatorDescriptor(IClusterController clusterController, Map<String, String> jobConfMap,
+ JobSpecification spec, String fileSystemURL, String inputFormatClassName, RecordDescriptor recordDescriptor) {
+ super(spec, null, recordDescriptor);
+ HadoopAdapter hadoopAdapter = HadoopAdapter.getInstance(fileSystemURL);
+ String inputPathString = jobConfMap.get("mapred.input.dir");
+ String[] inputPaths = inputPathString.split(",");
+ Map<String, List<HadoopFileSplit>> blocksToRead = hadoopAdapter.getInputSplits(inputPaths);
+ List<HadoopFileSplit> hadoopFileSplits = new ArrayList<HadoopFileSplit>();
+ for (String filePath : blocksToRead.keySet()) {
+ hadoopFileSplits.addAll(blocksToRead.get(filePath));
+ }
+ for (HadoopFileSplit hadoopFileSplit : hadoopFileSplits) {
+ System.out.println(" Hadoop File Split : " + hadoopFileSplit);
+ }
+ super.splits = hadoopFileSplits.toArray(new HadoopFileSplit[] {});
+ configurePartitionConstraints(clusterController, blocksToRead);
+ this.inputFormatClassName = inputFormatClassName;
+ this.jobConfMap = jobConfMap;
+ }
+
+ private void configurePartitionConstraints(IClusterController clusterController,
+ Map<String, List<HadoopFileSplit>> blocksToRead) {
+ List<LocationConstraint> locationConstraints = new ArrayList<LocationConstraint>();
+ Map<String, INodeController> registry = null;
+ try {
+ registry = clusterController.getRegistry();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ Map<String, String> hostnameToNodeIdMap = new HashMap<String, String>();
+ NCConfig ncConfig = null;
+ for (String nodeId : registry.keySet()) {
+ try {
+ ncConfig = ((INodeController) registry.get(nodeId)).getConfiguration();
+ String ipAddress = ncConfig.dataIPAddress;
+ String hostname = InetAddress.getByName(ipAddress).getHostName();
+ System.out.println(" hostname :" + hostname + " nodeid:" + nodeId);
+ hostnameToNodeIdMap.put(hostname, nodeId);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (String filePath : blocksToRead.keySet()) {
+ List<HadoopFileSplit> hadoopFileSplits = blocksToRead.get(filePath);
+ for (HadoopFileSplit hadoopFileSplit : hadoopFileSplits) {
+ String hostname = hadoopFileSplit.getHosts()[0];
+ System.out.println("host name is :" + hostname);
+ InetAddress address = null;
+ try {
+ address = InetAddress.getByName(hostname);
+ if (address.isLoopbackAddress()) {
+ Enumeration<NetworkInterface> netInterfaces = NetworkInterface.getNetworkInterfaces();
+ while (netInterfaces.hasMoreElements()) {
+ NetworkInterface ni = netInterfaces.nextElement();
+ InetAddress inetAddress = (InetAddress) ni.getInetAddresses().nextElement();
+ if (!inetAddress.isLoopbackAddress()) {
+ address = inetAddress;
+ break;
+ }
+ }
+ }
+ hostname = address.getHostName();
+ System.out.println("cannonical host name hyracks :" + hostname);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ String nodeId = hostnameToNodeIdMap.get(hostname);
+ System.out.println(" corresponding node id is :" + nodeId);
+ LocationConstraint locationConstraint = new AbsoluteLocationConstraint(nodeId);
+ locationConstraints.add(locationConstraint);
+ }
+ }
+
+ PartitionConstraint partitionConstraint = new PartitionConstraint(locationConstraints
+ .toArray(new LocationConstraint[] {}));
+ this.setPartitionConstraint(partitionConstraint);
+ }
+
+ @Override
+ protected IRecordReader createRecordReader(HadoopFileSplit fileSplit, RecordDescriptor desc) throws Exception {
+ Reporter reporter = createReporter();
+ IRecordReader recordReader = new HDFSCustomReader(jobConfMap, fileSplit, inputFormatClassName, reporter);
+ return recordReader;
+ }
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/HadoopReducerOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/HadoopReducerOperatorDescriptor.java
new file mode 100644
index 0000000..7881d45
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/HadoopReducerOperatorDescriptor.java
@@ -0,0 +1,280 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.hadoop;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.Counters.Counter;
+
+import edu.uci.ics.hyracks.api.dataflow.IDataReader;
+import edu.uci.ics.hyracks.api.dataflow.IDataWriter;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePullable;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.IOpenableDataWriterOperator;
+import edu.uci.ics.hyracks.coreops.group.IGroupAggregator;
+import edu.uci.ics.hyracks.coreops.group.PreclusteredGroupOperator;
+import edu.uci.ics.hyracks.coreops.hadoop.data.KeyComparatorFactory;
+import edu.uci.ics.hyracks.coreops.hadoop.data.WritableComparingComparatorFactory;
+import edu.uci.ics.hyracks.coreops.util.DeserializedOperatorNodePushable;
+import edu.uci.ics.hyracks.hadoop.util.ClasspathBasedHadoopClassFactory;
+import edu.uci.ics.hyracks.hadoop.util.DatatypeHelper;
+import edu.uci.ics.hyracks.hadoop.util.IHadoopClassFactory;
+
+public class HadoopReducerOperatorDescriptor<K2, V2, K3, V3> extends AbstractHadoopOperatorDescriptor {
+ private class ReducerAggregator implements IGroupAggregator {
+ private Reducer<K2, V2, K3, V3> reducer;
+ private DataWritingOutputCollector<K3, V3> output;
+ private Reporter reporter;
+
+ public ReducerAggregator(Reducer<K2, V2, K3, V3> reducer) {
+ this.reducer = reducer;
+ reducer.configure(getJobConf());
+ output = new DataWritingOutputCollector<K3, V3>();
+ reporter = new Reporter() {
+
+ @Override
+ public void progress() {
+
+ }
+
+ @Override
+ public void setStatus(String arg0) {
+
+ }
+
+ @Override
+ public void incrCounter(String arg0, String arg1, long arg2) {
+
+ }
+
+ @Override
+ public void incrCounter(Enum<?> arg0, long arg1) {
+
+ }
+
+ @Override
+ public InputSplit getInputSplit() throws UnsupportedOperationException {
+ return null;
+ }
+
+ @Override
+ public Counter getCounter(String arg0, String arg1) {
+ return null;
+ }
+
+ @Override
+ public Counter getCounter(Enum<?> arg0) {
+ return null;
+ }
+ };
+ }
+
+ @Override
+ public void aggregate(IDataReader<Object[]> reader, IDataWriter<Object[]> writer) throws HyracksDataException {
+
+ ValueIterator i = new ValueIterator();
+ i.reset(reader);
+ output.setWriter(writer);
+ try {
+
+ // -- - reduce - --
+ reducer.reduce(i.getKey(), i, output, reporter);
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ // -- - close - --
+ try {
+ reducer.close();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+
+ private class ValueIterator implements Iterator<V2> {
+ private IDataReader<Object[]> reader;
+ private K2 key;
+ private V2 value;
+
+ public K2 getKey() {
+ return key;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (value == null) {
+ Object[] tuple;
+ try {
+ tuple = reader.readData();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ if (tuple != null) {
+ value = (V2) tuple[1];
+ }
+ }
+ return value != null;
+ }
+
+ @Override
+ public V2 next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ V2 v = value;
+ value = null;
+ return v;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ void reset(IDataReader<Object[]> reader) {
+ this.reader = reader;
+ try {
+ Object[] tuple = reader.readData();
+ key = (K2) tuple[0];
+ value = (V2) tuple[1];
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static final long serialVersionUID = 1L;
+ private Class<? extends Reducer> reducerClass;
+ private static final String reducerClassKey = "mapred.reducer.class";
+ private static final String comparatorClassKey = "mapred.output.value.groupfn.class";
+ private IComparatorFactory comparatorFactory;
+
+
+ public HadoopReducerOperatorDescriptor(JobSpecification spec, IComparatorFactory comparatorFactory,
+ Class<? extends Reducer> reducerClass, RecordDescriptor recordDescriptor, JobConf jobConf) {
+ super(spec, recordDescriptor, jobConf, new ClasspathBasedHadoopClassFactory());
+ this.comparatorFactory = comparatorFactory;
+ this.reducerClass = reducerClass;
+ }
+
+
+ public HadoopReducerOperatorDescriptor(JobSpecification spec, JobConf conf, IHadoopClassFactory classFactory) {
+ super(spec, null , conf, classFactory);
+ }
+
+ private Reducer<K2, V2, K3, V3> createReducer() throws Exception {
+ if(reducerClass != null){
+ return reducerClass.newInstance();
+ }else{
+ Object reducer = getHadoopClassFactory().createReducer(getJobConfMap().get(reducerClassKey));
+ reducerClass = (Class<? extends Reducer>) reducer.getClass();
+ return (Reducer)reducer;
+ }
+ }
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ try {
+ if(this.comparatorFactory == null){
+ String comparatorClassName = getJobConfMap().get(comparatorClassKey);
+ RawComparator rawComparator = null;
+ if(comparatorClassName != null){
+ Class comparatorClazz = getHadoopClassFactory().loadClass(comparatorClassName);
+ this.comparatorFactory = new KeyComparatorFactory(comparatorClazz);
+
+ }else{
+ String mapOutputKeyClass = getJobConfMap().get("mapred.mapoutput.key.class");
+ if(getHadoopClassFactory() != null){
+ rawComparator = WritableComparator.get(getHadoopClassFactory().loadClass(mapOutputKeyClass));
+ }else{
+ rawComparator = WritableComparator.get((Class<? extends WritableComparable>)Class.forName(mapOutputKeyClass));
+ }
+ this.comparatorFactory = new WritableComparingComparatorFactory(rawComparator.getClass());
+ }
+ }
+ IOpenableDataWriterOperator op = new PreclusteredGroupOperator(new int[] { 0 },
+ new IComparator[] { comparatorFactory.createComparator() }, new ReducerAggregator(createReducer()));
+ return new DeserializedOperatorNodePushable(ctx, op, plan, getActivityNodeId());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public Class<? extends Reducer> getReducerClass() {
+ return reducerClass;
+ }
+
+ public void setReducerClass(Class<? extends Reducer> reducerClass) {
+ this.reducerClass = reducerClass;
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+
+ @Override
+ public RecordDescriptor getRecordDescriptor(JobConf conf) {
+ String outputKeyClassName = conf.get("mapred.output.key.class");
+ String outputValueClassName = conf.get("mapred.output.value.class");
+ RecordDescriptor recordDescriptor = null;
+ try{
+ if(getHadoopClassFactory() == null){
+ recordDescriptor = DatatypeHelper.createKeyValueRecordDescriptor((Class<? extends Writable>)Class.forName(outputKeyClassName), (Class<? extends Writable>)Class.forName(outputValueClassName));
+ }else{
+ recordDescriptor = DatatypeHelper.createKeyValueRecordDescriptor((Class<? extends Writable>)getHadoopClassFactory().loadClass(outputKeyClassName), (Class<? extends Writable>)getHadoopClassFactory().loadClass(outputValueClassName));
+ }
+ }catch(Exception e){
+ e.printStackTrace();
+ return null;
+ }
+ return recordDescriptor;
+ }
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/AbstractClassBasedDelegate.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/AbstractClassBasedDelegate.java
new file mode 100644
index 0000000..afc8ccb
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/AbstractClassBasedDelegate.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.hadoop.data;
+
+import java.io.ObjectStreamException;
+import java.io.Serializable;
+
+public class AbstractClassBasedDelegate<T> implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private Class<? extends T> klass;
+ protected transient T instance;
+
+ public AbstractClassBasedDelegate(Class<? extends T> klass) {
+ this.klass = klass;
+ init();
+ }
+
+ protected Object readResolve() throws ObjectStreamException {
+ init();
+ return this;
+ }
+
+ private void init() {
+ try {
+ instance = klass.newInstance();
+ } catch (InstantiationException e) {
+ throw new RuntimeException(e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/HadoopHashTuplePartitionComputerFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/HadoopHashTuplePartitionComputerFactory.java
new file mode 100644
index 0000000..e361ebb
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/HadoopHashTuplePartitionComputerFactory.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.hadoop.data;
+
+import java.io.DataInputStream;
+
+import org.apache.hadoop.io.Writable;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.comm.util.ByteBufferInputStream;
+
+public class HadoopHashTuplePartitionComputerFactory<K extends Writable> implements ITuplePartitionComputerFactory {
+ private static final long serialVersionUID = 1L;
+ private final ISerializerDeserializer<K> keyIO;
+
+ public HadoopHashTuplePartitionComputerFactory(ISerializerDeserializer<K> keyIO) {
+ this.keyIO = keyIO;
+ }
+
+ @Override
+ public ITuplePartitionComputer createPartitioner() {
+ return new ITuplePartitionComputer() {
+ private final ByteBufferInputStream bbis = new ByteBufferInputStream();
+ private final DataInputStream dis = new DataInputStream(bbis);
+
+ @Override
+ public int partition(FrameTupleAccessor accessor, int tIndex, int nParts) throws HyracksDataException {
+ int keyStart = accessor.getTupleStartOffset(tIndex) + accessor.getFieldSlotsLength()
+ + accessor.getFieldStartOffset(tIndex, 0);
+ bbis.setByteBuffer(accessor.getBuffer(), keyStart);
+ K key = keyIO.deserialize(dis);
+ int h = key.hashCode();
+ if (h < 0) {
+ h = -h;
+ }
+ return h % nParts;
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/HadoopPartitionerTuplePartitionComputerFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/HadoopPartitionerTuplePartitionComputerFactory.java
new file mode 100644
index 0000000..a80682f
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/HadoopPartitionerTuplePartitionComputerFactory.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.hadoop.data;
+
+import java.io.DataInputStream;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.Partitioner;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.comm.util.ByteBufferInputStream;
+
+public class HadoopPartitionerTuplePartitionComputerFactory<K extends Writable, V extends Writable> extends
+ AbstractClassBasedDelegate<Partitioner<K, V>> implements ITuplePartitionComputerFactory {
+ private static final long serialVersionUID = 1L;
+ private final ISerializerDeserializer<K> keyIO;
+ private final ISerializerDeserializer<V> valueIO;
+
+ public HadoopPartitionerTuplePartitionComputerFactory(Class<? extends Partitioner<K, V>> klass,
+ ISerializerDeserializer<K> keyIO, ISerializerDeserializer<V> valueIO) {
+ super(klass);
+ this.keyIO = keyIO;
+ this.valueIO = valueIO;
+ }
+
+ @Override
+ public ITuplePartitionComputer createPartitioner() {
+ return new ITuplePartitionComputer() {
+ private final ByteBufferInputStream bbis = new ByteBufferInputStream();
+ private final DataInputStream dis = new DataInputStream(bbis);
+
+ @Override
+ public int partition(FrameTupleAccessor accessor, int tIndex, int nParts) throws HyracksDataException {
+ int keyStart = accessor.getTupleStartOffset(tIndex) + accessor.getFieldSlotsLength()
+ + accessor.getFieldStartOffset(tIndex, 0);
+ bbis.setByteBuffer(accessor.getBuffer(), keyStart);
+ K key = keyIO.deserialize(dis);
+ int valueStart = accessor.getTupleStartOffset(tIndex) + accessor.getFieldSlotsLength()
+ + accessor.getFieldStartOffset(tIndex, 1);
+ bbis.setByteBuffer(accessor.getBuffer(), valueStart);
+ V value = valueIO.deserialize(dis);
+ return instance.getPartition(key, value, nParts);
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/KeyBinaryComparatorFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/KeyBinaryComparatorFactory.java
new file mode 100644
index 0000000..8b16fc4
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/KeyBinaryComparatorFactory.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.hadoop.data;
+
+import org.apache.hadoop.io.RawComparator;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.util.ReflectionUtils;
+
+public class KeyBinaryComparatorFactory<T> implements IBinaryComparatorFactory {
+ private static final long serialVersionUID = 1L;
+
+ private Class<? extends RawComparator<T>> cmpClass;
+
+ public KeyBinaryComparatorFactory(Class<? extends RawComparator<T>> cmpClass) {
+ this.cmpClass = cmpClass;
+ }
+
+ @Override
+ public IBinaryComparator createBinaryComparator() {
+ final RawComparator<T> instance = ReflectionUtils.createInstance(cmpClass);
+ return new IBinaryComparator() {
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ return instance.compare(b1, s1, l1, b2, s2, l2);
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/KeyComparatorFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/KeyComparatorFactory.java
new file mode 100644
index 0000000..9f48251
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/KeyComparatorFactory.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.hadoop.data;
+
+import org.apache.hadoop.io.RawComparator;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IComparatorFactory;
+import edu.uci.ics.hyracks.util.ReflectionUtils;
+
+public class KeyComparatorFactory<T> implements IComparatorFactory<T> {
+ private static final long serialVersionUID = 1L;
+ private Class<? extends RawComparator<T>> cmpClass;
+
+ public KeyComparatorFactory(Class<? extends RawComparator<T>> cmpClass) {
+ this.cmpClass = cmpClass;
+ }
+
+ @Override
+ public IComparator<T> createComparator() {
+ final RawComparator<T> instance = ReflectionUtils.createInstance(cmpClass);
+ return new IComparator<T>() {
+ @Override
+ public int compare(T o1, T o2) {
+ return instance.compare(o1, o2);
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/WritableComparingBinaryComparatorFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/WritableComparingBinaryComparatorFactory.java
new file mode 100644
index 0000000..b633742
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/WritableComparingBinaryComparatorFactory.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.hadoop.data;
+
+import org.apache.hadoop.io.WritableComparator;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.util.ReflectionUtils;
+
+public class WritableComparingBinaryComparatorFactory<T> implements IBinaryComparatorFactory {
+ private static final long serialVersionUID = 1L;
+
+ private Class<? extends WritableComparator> cmpClass;
+
+ public WritableComparingBinaryComparatorFactory(Class<? extends WritableComparator> cmpClass) {
+ this.cmpClass = cmpClass;
+ }
+
+ @Override
+ public IBinaryComparator createBinaryComparator() {
+ final WritableComparator instance = ReflectionUtils.createInstance(cmpClass);
+ return new IBinaryComparator() {
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ return instance.compare(b1, s1, l1, b2, s2, l2);
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/WritableComparingComparatorFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/WritableComparingComparatorFactory.java
new file mode 100644
index 0000000..c53abcc
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/hadoop/data/WritableComparingComparatorFactory.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.hadoop.data;
+
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IComparatorFactory;
+import edu.uci.ics.hyracks.util.ReflectionUtils;
+
+public class WritableComparingComparatorFactory<T> implements IComparatorFactory<WritableComparable<T>> {
+ private Class<? extends WritableComparator> klass;
+
+ public WritableComparingComparatorFactory(Class<? extends WritableComparator> klass) {
+ this.klass = klass;
+ }
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IComparator<WritableComparable<T>> createComparator() {
+ final WritableComparator instance = ReflectionUtils.createInstance(klass);
+ return new IComparator<WritableComparable<T>>() {
+ @Override
+ public int compare(WritableComparable<T> o1, WritableComparable<T> o2) {
+ return instance.compare(o1, o2);
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/join/InMemoryHashJoin.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/join/InMemoryHashJoin.java
new file mode 100644
index 0000000..0962895
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/join/InMemoryHashJoin.java
@@ -0,0 +1,131 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.join;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.comm.io.FrameTuplePairComparator;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public class InMemoryHashJoin {
+ private final Link[] table;
+ private final List<ByteBuffer> buffers;
+ private final FrameTupleAccessor accessor0;
+ private final ITuplePartitionComputer tpc0;
+ private final FrameTupleAccessor accessor1;
+ private final ITuplePartitionComputer tpc1;
+ private final FrameTupleAppender appender;
+ private final FrameTuplePairComparator tpComparator;
+ private final ByteBuffer outBuffer;
+
+ public InMemoryHashJoin(HyracksContext ctx, int tableSize, FrameTupleAccessor accessor0,
+ ITuplePartitionComputer tpc0, FrameTupleAccessor accessor1, ITuplePartitionComputer tpc1,
+ FrameTuplePairComparator comparator) {
+ table = new Link[tableSize];
+ buffers = new ArrayList<ByteBuffer>();
+ this.accessor0 = accessor0;
+ this.tpc0 = tpc0;
+ this.accessor1 = accessor1;
+ this.tpc1 = tpc1;
+ appender = new FrameTupleAppender(ctx);
+ tpComparator = comparator;
+ outBuffer = ctx.getResourceManager().allocateFrame();
+ appender.reset(outBuffer, true);
+ }
+
+ public void build(ByteBuffer buffer) throws HyracksDataException {
+ buffers.add(buffer);
+ int bIndex = buffers.size() - 1;
+ accessor0.reset(buffer);
+ int tCount = accessor0.getTupleCount();
+ for (int i = 0; i < tCount; ++i) {
+ int entry = tpc0.partition(accessor0, i, table.length);
+ long tPointer = (((long) bIndex) << 32) + i;
+ Link link = table[entry];
+ if (link == null) {
+ link = table[entry] = new Link();
+ }
+ link.add(tPointer);
+ }
+ }
+
+ public void join(ByteBuffer buffer, IFrameWriter writer) throws HyracksDataException {
+ accessor1.reset(buffer);
+ int tupleCount1 = accessor1.getTupleCount();
+ for (int i = 0; i < tupleCount1; ++i) {
+ int entry = tpc1.partition(accessor1, i, table.length);
+ Link link = table[entry];
+ if (link != null) {
+ for (int j = 0; j < link.size; ++j) {
+ long pointer = link.pointers[j];
+ int bIndex = (int) ((pointer >> 32) & 0xffffffff);
+ int tIndex = (int) (pointer & 0xffffffff);
+ accessor0.reset(buffers.get(bIndex));
+ int c = tpComparator.compare(accessor0, tIndex, accessor1, i);
+ if (c == 0) {
+ if (!appender.appendConcat(accessor0, tIndex, accessor1, i)) {
+ flushFrame(outBuffer, writer);
+ appender.reset(outBuffer, true);
+ if (!appender.appendConcat(accessor0, tIndex, accessor1, i)) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ public void closeJoin(IFrameWriter writer) throws HyracksDataException {
+ if (appender.getTupleCount() > 0) {
+ flushFrame(outBuffer, writer);
+ }
+ }
+
+ private void flushFrame(ByteBuffer buffer, IFrameWriter writer) throws HyracksDataException {
+ buffer.position(0);
+ buffer.limit(buffer.capacity());
+ writer.nextFrame(outBuffer);
+ buffer.position(0);
+ buffer.limit(buffer.capacity());
+ }
+
+ private static class Link {
+ private static final int INIT_POINTERS_SIZE = 8;
+
+ long[] pointers;
+ int size;
+
+ Link() {
+ pointers = new long[INIT_POINTERS_SIZE];
+ size = 0;
+ }
+
+ void add(long pointer) {
+ if (size >= pointers.length) {
+ pointers = Arrays.copyOf(pointers, pointers.length * 2);
+ }
+ pointers[size++] = pointer;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/join/InMemoryHashJoinOperatorDescriptor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/join/InMemoryHashJoinOperatorDescriptor.java
new file mode 100644
index 0000000..38fbc69
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/join/InMemoryHashJoinOperatorDescriptor.java
@@ -0,0 +1,208 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.join;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePullable;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.comm.io.FrameTuplePairComparator;
+import edu.uci.ics.hyracks.comm.util.FrameUtils;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.coreops.base.AbstractActivityNode;
+import edu.uci.ics.hyracks.coreops.base.AbstractOperatorDescriptor;
+
+public class InMemoryHashJoinOperatorDescriptor extends AbstractOperatorDescriptor {
+ private static final String JOINER = "joiner";
+
+ private static final long serialVersionUID = 1L;
+ private final int[] keys0;
+ private final int[] keys1;
+ private final IBinaryHashFunctionFactory[] hashFunctionFactories;
+ private final IBinaryComparatorFactory[] comparatorFactories;
+ private final int tableSize;
+
+ public InMemoryHashJoinOperatorDescriptor(JobSpecification spec, int[] keys0, int[] keys1,
+ IBinaryHashFunctionFactory[] hashFunctionFactories, IBinaryComparatorFactory[] comparatorFactories,
+ RecordDescriptor recordDescriptor, int tableSize) {
+ super(spec, 2, 1);
+ this.keys0 = keys0;
+ this.keys1 = keys1;
+ this.hashFunctionFactories = hashFunctionFactories;
+ this.comparatorFactories = comparatorFactories;
+ this.tableSize = tableSize;
+ recordDescriptors[0] = recordDescriptor;
+ }
+
+ @Override
+ public void contributeTaskGraph(IActivityGraphBuilder builder) {
+ HashBuildActivityNode hba = new HashBuildActivityNode();
+ HashProbeActivityNode hpa = new HashProbeActivityNode();
+
+ builder.addTask(hba);
+ builder.addSourceEdge(0, hba, 0);
+
+ builder.addTask(hpa);
+ builder.addSourceEdge(1, hpa, 0);
+ builder.addTargetEdge(0, hpa, 0);
+
+ builder.addBlockingEdge(hba, hpa);
+ }
+
+ private class HashBuildActivityNode extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final HyracksContext ctx, JobPlan plan,
+ final IOperatorEnvironment env, int partition) {
+ final RecordDescriptor rd0 = plan.getJobSpecification()
+ .getOperatorInputRecordDescriptor(getOperatorId(), 0);
+ final RecordDescriptor rd1 = plan.getJobSpecification()
+ .getOperatorInputRecordDescriptor(getOperatorId(), 1);
+ final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
+ for (int i = 0; i < comparatorFactories.length; ++i) {
+ comparators[i] = comparatorFactories[i].createBinaryComparator();
+ }
+ IOperatorNodePushable op = new IOperatorNodePushable() {
+ private InMemoryHashJoin joiner;
+
+ @Override
+ public void open() throws HyracksDataException {
+ ITuplePartitionComputer hpc0 = new FieldHashPartitionComputerFactory(keys0, hashFunctionFactories)
+ .createPartitioner();
+ ITuplePartitionComputer hpc1 = new FieldHashPartitionComputerFactory(keys1, hashFunctionFactories)
+ .createPartitioner();
+ joiner = new InMemoryHashJoin(ctx, tableSize, new FrameTupleAccessor(ctx, rd0), hpc0,
+ new FrameTupleAccessor(ctx, rd1), hpc1, new FrameTuplePairComparator(keys0, keys1,
+ comparators));
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ ByteBuffer copyBuffer = ctx.getResourceManager().allocateFrame();
+ FrameUtils.copy(buffer, copyBuffer);
+ joiner.build(copyBuffer);
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ env.set(JOINER, joiner);
+ }
+
+ @Override
+ public void setFrameWriter(int index, IFrameWriter writer) {
+ throw new IllegalArgumentException();
+ }
+ };
+ return op;
+ }
+
+ @Override
+ public IOperatorDescriptor getOwner() {
+ return InMemoryHashJoinOperatorDescriptor.this;
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+ }
+
+ private class HashProbeActivityNode extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IOperatorNodePullable createPullRuntime(HyracksContext ctx, JobPlan plan, IOperatorEnvironment env,
+ int partition) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final HyracksContext ctx, JobPlan plan,
+ final IOperatorEnvironment env, int partition) {
+ IOperatorNodePushable op = new IOperatorNodePushable() {
+ private IFrameWriter writer;
+ private InMemoryHashJoin joiner;
+
+ @Override
+ public void open() throws HyracksDataException {
+ joiner = (InMemoryHashJoin) env.get(JOINER);
+ writer.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ joiner.join(buffer, writer);
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ joiner.closeJoin(writer);
+ writer.close();
+ env.set(JOINER, null);
+ }
+
+ @Override
+ public void setFrameWriter(int index, IFrameWriter writer) {
+ if (index != 0) {
+ throw new IllegalStateException();
+ }
+ this.writer = writer;
+ }
+ };
+ return op;
+ }
+
+ @Override
+ public IOperatorDescriptor getOwner() {
+ return InMemoryHashJoinOperatorDescriptor.this;
+ }
+
+ @Override
+ public boolean supportsPullInterface() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsPushInterface() {
+ return true;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/map/ReflectionBasedMapperFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/map/ReflectionBasedMapperFactory.java
new file mode 100644
index 0000000..c284ff1
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/map/ReflectionBasedMapperFactory.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.map;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.coreops.IMapper;
+import edu.uci.ics.hyracks.coreops.IMapperFactory;
+
+public class ReflectionBasedMapperFactory implements IMapperFactory {
+ private static final long serialVersionUID = 1L;
+
+ private final Class<? extends IMapper> mapperClass;
+
+ public ReflectionBasedMapperFactory(Class<? extends IMapper> mapperClass) {
+ this.mapperClass = mapperClass;
+ }
+
+ @Override
+ public IMapper createMapper() throws HyracksDataException {
+ try {
+ return mapperClass.newInstance();
+ } catch (InstantiationException e) {
+ throw new HyracksDataException(e);
+ } catch (IllegalAccessException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/map/SamplerMapFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/map/SamplerMapFactory.java
new file mode 100644
index 0000000..44541a1
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/map/SamplerMapFactory.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.map;
+
+import java.util.Random;
+
+import edu.uci.ics.hyracks.api.dataflow.IDataWriter;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.coreops.IMapper;
+import edu.uci.ics.hyracks.coreops.IMapperFactory;
+
+public class SamplerMapFactory implements IMapperFactory {
+ private static final long serialVersionUID = 1L;
+ private final float sampleRatio;
+
+ public SamplerMapFactory(float sampleRatio) {
+ this.sampleRatio = sampleRatio;
+ }
+
+ @Override
+ public IMapper createMapper() throws HyracksDataException {
+ return new IMapper() {
+ private Random random = new Random();
+
+ @Override
+ public void map(Object[] data, IDataWriter<Object[]> writer) throws HyracksDataException {
+ if (random.nextFloat() <= sampleRatio) {
+ writer.writeData(data);
+ }
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/DeserializedOperatorNodePushable.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/DeserializedOperatorNodePushable.java
new file mode 100644
index 0000000..610cc51
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/DeserializedOperatorNodePushable.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.util;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.ActivityNodeId;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.comm.io.FrameDeserializer;
+import edu.uci.ics.hyracks.comm.io.SerializingDataWriter;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.base.IOpenableDataWriterOperator;
+
+public final class DeserializedOperatorNodePushable implements IOperatorNodePushable {
+ private final HyracksContext ctx;
+
+ private final IOpenableDataWriterOperator delegate;
+
+ private final JobPlan plan;
+
+ private final ActivityNodeId aid;
+
+ private final FrameDeserializer deserializer;
+
+ public DeserializedOperatorNodePushable(HyracksContext ctx, IOpenableDataWriterOperator delegate, JobPlan plan,
+ ActivityNodeId aid) {
+ this.ctx = ctx;
+ this.delegate = delegate;
+ this.plan = plan;
+ this.aid = aid;
+ List<Integer> inList = plan.getTaskInputMap().get(aid);
+ deserializer = inList == null ? null : new FrameDeserializer(ctx, plan.getTaskInputRecordDescriptor(aid, 0));
+ }
+
+ @Override
+ public void setFrameWriter(int index, IFrameWriter writer) {
+ delegate.setDataWriter(index, new SerializingDataWriter(ctx, plan.getTaskOutputRecordDescriptor(aid, index),
+ writer));
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ delegate.close();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ deserializer.reset(buffer);
+ while (!deserializer.done()) {
+ delegate.writeData(deserializer.deserializeRecord());
+ }
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ delegate.open();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/ReferenceEntry.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/ReferenceEntry.java
new file mode 100644
index 0000000..025747d
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/ReferenceEntry.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.util;
+
+import edu.uci.ics.hyracks.comm.io.FrameTupleAccessor;
+
+public class ReferenceEntry {
+ private final int runid;
+ private FrameTupleAccessor acccessor;
+ private int tupleIndex;
+
+ public ReferenceEntry(int runid, FrameTupleAccessor fta, int tupleIndex) {
+ super();
+ this.runid = runid;
+ this.acccessor = fta;
+ this.tupleIndex = tupleIndex;
+ }
+
+ public int getRunid() {
+ return runid;
+ }
+
+ public FrameTupleAccessor getAccessor() {
+ return acccessor;
+ }
+
+ public void setAccessor(FrameTupleAccessor fta) {
+ this.acccessor = fta;
+ }
+
+ public int getTupleIndex() {
+ return tupleIndex;
+ }
+
+ public void setTupleIndex(int tupleIndex) {
+ this.tupleIndex = tupleIndex;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/ReferencedPriorityQueue.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/ReferencedPriorityQueue.java
new file mode 100644
index 0000000..f64320b
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/ReferencedPriorityQueue.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.util;
+
+import java.io.IOException;
+import java.util.BitSet;
+import java.util.Comparator;
+
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public class ReferencedPriorityQueue {
+ private final HyracksContext ctx;
+ private final RecordDescriptor recordDescriptor;
+ private final ReferenceEntry entries[];
+ private final int size;
+ private final BitSet runAvail;
+ private int nItems;
+
+ private final Comparator<ReferenceEntry> comparator;
+
+ public ReferencedPriorityQueue(HyracksContext ctx, RecordDescriptor recordDescriptor, int initSize,
+ Comparator<ReferenceEntry> comparator) throws IOException {
+ this.ctx = ctx;
+ this.recordDescriptor = recordDescriptor;
+ if (initSize < 1)
+ throw new IllegalArgumentException();
+ this.comparator = comparator;
+ nItems = initSize;
+ size = (initSize + 1) & 0xfffffffe;
+ entries = new ReferenceEntry[size];
+ runAvail = new BitSet(size);
+ runAvail.set(0, initSize, true);
+ for (int i = 0; i < size; i++) {
+ entries[i] = new ReferenceEntry(i, null, -1);
+ }
+ }
+
+ /**
+ * Retrieve the top entry without removing it
+ *
+ * @return the top entry
+ */
+ public ReferenceEntry peek() {
+ return entries[0];
+ }
+
+ /**
+ * compare the new entry with entries within the queue, to find a spot for
+ * this new entry
+ *
+ * @param entry
+ * @return runid of this entry
+ * @throws IOException
+ */
+ public int popAndReplace(FrameTupleAccessor fta, int tIndex) throws IOException {
+ ReferenceEntry entry = entries[0];
+ if (entry.getAccessor() == null) {
+ entry.setAccessor(new FrameTupleAccessor(ctx, recordDescriptor));
+ }
+ entry.getAccessor().reset(fta.getBuffer());
+ entry.setTupleIndex(tIndex);
+
+ add(entry);
+ return entry.getRunid();
+ }
+
+ /**
+ * Push entry into priority queue
+ *
+ * @param e
+ * the new Entry
+ */
+ private void add(ReferenceEntry e) {
+ ReferenceEntry min = entries[0];
+ int slot = (size >> 1) + (min.getRunid() >> 1);
+
+ ReferenceEntry curr = e;
+ while (!runAvail.isEmpty() && slot > 0) {
+ int c = 0;
+ if (!runAvail.get(entries[slot].getRunid())) {
+ // run of entries[slot] is exhausted, i.e. not available, curr
+ // wins
+ c = 1;
+ } else if (entries[slot].getAccessor() != null /*
+ * entries[slot] is
+ * not MIN value
+ */
+ && runAvail.get(curr.getRunid() /* curr run is available */)) {
+
+ if (curr.getAccessor() != null) {
+ c = comparator.compare(entries[slot], curr);
+ } else {
+ // curr is MIN value, wins
+ c = 1;
+ }
+ }
+
+ if (c <= 0) { // curr lost
+ // entries[slot] swaps up
+ ReferenceEntry tmp = entries[slot];
+ entries[slot] = curr;
+ curr = tmp;// winner to pass up
+ }// else curr wins
+ slot >>= 1;
+ }
+ // set new entries[0]
+ entries[0] = curr;
+ }
+
+ /**
+ * Pop is called only when a run is exhausted
+ *
+ * @return
+ */
+ public ReferenceEntry pop() {
+ ReferenceEntry min = entries[0];
+ runAvail.clear(min.getRunid());
+ add(min);
+ nItems--;
+ return min;
+ }
+
+ public boolean areRunsExhausted() {
+ return runAvail.isEmpty();
+ }
+
+ public int size() {
+ return nItems;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/SynchronizedBoundedBuffer.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/SynchronizedBoundedBuffer.java
new file mode 100644
index 0000000..f1d3d3c
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/SynchronizedBoundedBuffer.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.util;
+
+import java.util.Collection;
+
+public class SynchronizedBoundedBuffer<T> {
+ private static final int QUEUE_SIZE = 8192;
+ private Object[] buffer;
+ private int head;
+ private int tail;
+
+ public SynchronizedBoundedBuffer() {
+ buffer = new Object[QUEUE_SIZE];
+ head = 0;
+ tail = 0;
+ }
+
+ public synchronized void put(T o) throws InterruptedException {
+ while (full()) {
+ wait();
+ }
+ buffer[tail] = o;
+ tail = (tail + 1) % QUEUE_SIZE;
+ notifyAll();
+ }
+
+ public synchronized void putAll(Collection<? extends T> c) throws InterruptedException {
+ for (T o : c) {
+ while (full()) {
+ wait();
+ }
+ buffer[tail] = o;
+ tail = (tail + 1) % QUEUE_SIZE;
+ }
+ notifyAll();
+ }
+
+ public synchronized T get() throws InterruptedException {
+ while (empty()) {
+ wait();
+ }
+ T o = (T) buffer[head];
+ head = (head + 1) % QUEUE_SIZE;
+ notifyAll();
+ return o;
+ }
+
+ private boolean empty() {
+ return head == tail;
+ }
+
+ private boolean full() {
+ return (tail + 1) % QUEUE_SIZE == head;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/SynchronizedBoundedBufferDataReader.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/SynchronizedBoundedBufferDataReader.java
new file mode 100644
index 0000000..7599bdc
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/SynchronizedBoundedBufferDataReader.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.util;
+
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataReader;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public class SynchronizedBoundedBufferDataReader implements IOpenableDataReader<Object[]> {
+ private SynchronizedBoundedBuffer<Object[]> queue;
+
+ public SynchronizedBoundedBufferDataReader(SynchronizedBoundedBuffer<Object[]> queue) {
+ this.queue = queue;
+ }
+
+ @Override
+ public Object[] readData() throws HyracksDataException {
+ try {
+ return queue.get();
+ } catch (InterruptedException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void close() {
+ queue = null;
+ }
+
+ @Override
+ public void open() {
+ // do nothing
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/SynchronizedBoundedBufferDataWriter.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/SynchronizedBoundedBufferDataWriter.java
new file mode 100644
index 0000000..311ea57
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/coreops/util/SynchronizedBoundedBufferDataWriter.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.coreops.util;
+
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataWriter;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public class SynchronizedBoundedBufferDataWriter implements IOpenableDataWriter<Object[]> {
+ private SynchronizedBoundedBuffer<Object[]> queue;
+
+ public SynchronizedBoundedBufferDataWriter(SynchronizedBoundedBuffer<Object[]> queue) {
+ this.queue = queue;
+ }
+
+ @Override
+ public void writeData(Object[] data) throws HyracksDataException {
+ try {
+ queue.put(data);
+ } catch (InterruptedException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ try {
+ queue.put(null);
+ } catch (InterruptedException e) {
+ throw new HyracksDataException(e);
+ }
+ queue = null;
+ }
+
+ @Override
+ public void open() {
+ // do nothing
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/dataflow/base/IOperatorDescriptorVisitor.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/dataflow/base/IOperatorDescriptorVisitor.java
new file mode 100644
index 0000000..c01d183
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/dataflow/base/IOperatorDescriptorVisitor.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.base;
+
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+
+public interface IOperatorDescriptorVisitor {
+ public void visit(IOperatorDescriptor op) throws Exception;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/dataflow/util/PlanUtils.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/dataflow/util/PlanUtils.java
new file mode 100644
index 0000000..e22c60b
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/dataflow/util/PlanUtils.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.util;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.base.IOperatorDescriptorVisitor;
+
+public class PlanUtils {
+ public static void visit(JobSpecification spec, IOperatorDescriptorVisitor visitor) throws Exception {
+ Set<OperatorDescriptorId> seen = new HashSet<OperatorDescriptorId>();
+ for (IOperatorDescriptor op : spec.getOperatorMap().values()) {
+ visitOperator(visitor, seen, op);
+ }
+ }
+
+ private static void visitOperator(IOperatorDescriptorVisitor visitor, Set<OperatorDescriptorId> seen,
+ IOperatorDescriptor op) throws Exception {
+ if (!seen.contains(op)) {
+ visitor.visit(op);
+ }
+ seen.add(op.getOperatorId());
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/driver/CCDriver.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/driver/CCDriver.java
new file mode 100644
index 0000000..425f967
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/driver/CCDriver.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.driver;
+
+import org.kohsuke.args4j.CmdLineParser;
+
+import edu.uci.ics.hyracks.config.CCConfig;
+import edu.uci.ics.hyracks.controller.ClusterControllerService;
+
+public class CCDriver {
+ public static void main(String args[]) throws Exception {
+ CCConfig ccConfig = new CCConfig();
+ CmdLineParser cp = new CmdLineParser(ccConfig);
+ try {
+ cp.parseArgument(args);
+ } catch (Exception e) {
+ System.err.println(e.getMessage());
+ cp.printUsage(System.err);
+ return;
+ }
+ ClusterControllerService ccService = new ClusterControllerService(ccConfig);
+ ccService.start();
+ while (true) {
+ Thread.sleep(100000);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/driver/NCDriver.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/driver/NCDriver.java
new file mode 100644
index 0000000..657b304
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/driver/NCDriver.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.driver;
+
+import org.kohsuke.args4j.CmdLineParser;
+
+import edu.uci.ics.dcache.client.DCacheClient;
+import edu.uci.ics.dcache.client.DCacheClientConfig;
+import edu.uci.ics.hyracks.config.NCConfig;
+import edu.uci.ics.hyracks.controller.NodeControllerService;
+
+public class NCDriver {
+ public static void main(String args[]) throws Exception {
+ NCConfig ncConfig = new NCConfig();
+ CmdLineParser cp = new CmdLineParser(ncConfig);
+ try {
+ cp.parseArgument(args);
+ } catch (Exception e) {
+ System.err.println(e.getMessage());
+ cp.printUsage(System.err);
+ return;
+ }
+
+ DCacheClientConfig dccConfig = new DCacheClientConfig();
+ dccConfig.servers = ncConfig.dcacheClientServers;
+ dccConfig.serverLocal = ncConfig.dcacheClientServerLocal;
+ dccConfig.path = ncConfig.dcacheClientPath;
+
+ DCacheClient.get().init(dccConfig);
+
+ final NodeControllerService nService = new NodeControllerService(ncConfig);
+ nService.start();
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ nService.stop();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ while (true) {
+ Thread.sleep(10000);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/ClasspathBasedHadoopClassFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/ClasspathBasedHadoopClassFactory.java
new file mode 100644
index 0000000..e15ef8c
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/ClasspathBasedHadoopClassFactory.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.hadoop.util;
+
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.Reducer;
+
+public class ClasspathBasedHadoopClassFactory implements IHadoopClassFactory {
+
+ @Override
+ public Mapper createMapper(String mapClassName) throws Exception {
+ Class clazz = loadClass(mapClassName);
+ return (Mapper)clazz.newInstance();
+ }
+
+ @Override
+ public Reducer createReducer(String reduceClassName) throws Exception {
+ Class clazz = loadClass(reduceClassName);
+ return (Reducer)clazz.newInstance();
+ }
+
+ @Override
+ public Class loadClass(String className) throws Exception {
+ Class clazz = Class.forName(className);
+ return clazz;
+ }
+
+
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/DatatypeHelper.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/DatatypeHelper.java
new file mode 100644
index 0000000..199bd8d
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/DatatypeHelper.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.hadoop.util;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.JobConf;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+@SuppressWarnings("deprecation")
+public class DatatypeHelper {
+ private static final class WritableSerializerDeserializer<T extends Writable> implements ISerializerDeserializer<T> {
+ private static final long serialVersionUID = 1L;
+
+ private Class<T> clazz;
+
+ private WritableSerializerDeserializer(Class<T> clazz) {
+ this.clazz = clazz;
+ }
+
+ private T createInstance() throws HyracksDataException {
+ // TODO remove "if", create a new WritableInstanceOperations class
+ // that deals with Writables that don't have public constructors
+ if (NullWritable.class.equals(clazz)) {
+ return (T) NullWritable.get();
+ }
+ try {
+ return clazz.newInstance();
+ } catch (InstantiationException e) {
+ throw new HyracksDataException(e);
+ } catch (IllegalAccessException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public T deserialize(DataInput in) throws HyracksDataException {
+ T o = createInstance();
+ try {
+ o.readFields(in);
+ } catch (IOException e) {
+ e.printStackTrace();
+ // throw new HyracksDataException(e);
+ }
+ return o;
+ }
+
+ @Override
+ public void serialize(T instance, DataOutput out) throws HyracksDataException {
+ try {
+ instance.write(out);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+
+ public static ISerializerDeserializer<? extends Writable> createSerializerDeserializer(
+ Class<? extends Writable> fClass) {
+ return new WritableSerializerDeserializer(fClass);
+ }
+
+ public static RecordDescriptor createKeyValueRecordDescriptor(Class<? extends Writable> keyClass,
+ Class<? extends Writable> valueClass) {
+ ISerializerDeserializer[] fields = new ISerializerDeserializer[2];
+ fields[0] = createSerializerDeserializer(keyClass);
+ fields[1] = createSerializerDeserializer(valueClass);
+ return new RecordDescriptor(fields);
+ }
+
+ public static RecordDescriptor createOneFieldRecordDescriptor(Class<? extends Writable> fieldClass) {
+ ISerializerDeserializer[] fields = new ISerializerDeserializer[1];
+ fields[0] = createSerializerDeserializer(fieldClass);
+ return new RecordDescriptor(fields);
+ }
+
+ public static JobConf hashMap2JobConf(HashMap<String, String> jobConfMap) {
+ JobConf jobConf;
+ synchronized (Configuration.class) {
+ jobConf = new JobConf();
+ for (Entry<String, String> entry : jobConfMap.entrySet()) {
+ jobConf.set(entry.getKey(), entry.getValue());
+ }
+ }
+ return jobConf;
+ }
+
+ public static HashMap<String, String> jobConf2HashMap(JobConf jobConf) {
+ HashMap<String, String> jobConfMap = new HashMap<String, String>();
+ for (Entry<String, String> entry : jobConf) {
+ jobConfMap.put(entry.getKey(), entry.getValue());
+ }
+ return jobConfMap;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/DuplicateKeyMapper.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/DuplicateKeyMapper.java
new file mode 100644
index 0000000..3d15d58
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/DuplicateKeyMapper.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.hadoop.util;
+
+import java.util.Properties;
+
+import edu.uci.ics.hyracks.api.dataflow.IDataWriter;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.coreops.IMapper;
+
+public class DuplicateKeyMapper implements IMapper {
+
+ @Override
+ public void map(Object[] data, IDataWriter<Object[]> writer) throws HyracksDataException {
+ writer.writeData(new Object[] { data[0], data[1], data[0] });
+
+ }
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/HadoopAdapter.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/HadoopAdapter.java
new file mode 100644
index 0000000..a6edfc4
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/HadoopAdapter.java
@@ -0,0 +1,163 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.hadoop.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.util.StringUtils;
+
+import edu.uci.ics.hyracks.coreops.file.FileSplit;
+
+public class HadoopAdapter {
+
+ private static ClientProtocol namenode;
+ private static FileSystem fileSystem;
+ private static JobConf jobConf;
+ private static HadoopAdapter instance;
+
+ public static final String DFS_DATA_DIR = "dfs.data.dir";
+ public static final String FS_DEFAULT_NAME = "fs.default.name";
+ public static final String DFS_REPLICATION = "dfs.replication";
+
+ public static HadoopAdapter getInstance(String fileSystemURL){
+ if(instance == null){
+ jobConf = new JobConf(true);
+ String [] urlParts = parseURL(fileSystemURL);
+ jobConf.set(FS_DEFAULT_NAME, fileSystemURL);
+ instance = new HadoopAdapter(new InetSocketAddress(urlParts[1], Integer.parseInt(urlParts[2])));
+ }
+ return instance;
+ }
+
+ public static JobConf getConf() {
+ return jobConf;
+ }
+
+ private HadoopAdapter (InetSocketAddress address){
+ try{
+ this.namenode = getNameNode(address);
+ fileSystem = FileSystem.get(jobConf);
+ }catch(IOException ioe){
+ ioe.printStackTrace();
+ }
+ }
+
+ private static String[] parseURL(String urlString){
+ String[] urlComponents = urlString.split(":");
+ urlComponents[1] = urlComponents[1].substring(2);
+ return urlComponents;
+ }
+
+
+ public Map<String,List<HadoopFileSplit>> getInputSplits(String[] inputPaths){
+ List<HadoopFileSplit> hadoopFileSplits = new ArrayList<HadoopFileSplit>();
+ Path[] paths = new Path[inputPaths.length];
+ int index =0;
+ for(String inputPath : inputPaths){
+ paths[index++] = new Path(StringUtils.unEscapeString(inputPaths[0]));
+ }
+ Map<String,List<HadoopFileSplit>> fileSplitInfo = getBlocksToRead(paths);
+ return fileSplitInfo;
+ }
+
+ private static Map<String,List<HadoopFileSplit>> getBlocksToRead(Path[] inputPaths){
+ Map<String,List<HadoopFileSplit>> hadoopFileSplitMap = new HashMap<String,List<HadoopFileSplit>>();
+ for (int i=0;i<inputPaths.length;i++){
+ try{
+ String absolutePathPrint = getAbsolutePath(inputPaths[i]);
+ FileStatus[] fileStatuses = namenode.getListing(absolutePathPrint);
+ for(int j=0;j<fileStatuses.length;j++){
+ Path filePath = fileStatuses[j].getPath();
+ String absolutePath = getAbsolutePath(filePath);
+ List<HadoopFileSplit> fileSplits = getFileBlocks(absolutePath,fileStatuses[j]);
+ if(fileSplits!=null && fileSplits.size() > 0){
+ hadoopFileSplitMap.put(absolutePath, fileSplits);
+ }
+ }
+ }catch(IOException ioe){
+ ioe.printStackTrace();
+ }
+
+ }
+ return hadoopFileSplitMap;
+ }
+
+ private static ClientProtocol getNameNode(InetSocketAddress address) throws IOException{
+ return (ClientProtocol)getProtocol(ClientProtocol.class, address, new JobConf());
+ }
+
+ private static String getAbsolutePath(Path path){
+ StringBuffer absolutePath = new StringBuffer();
+ List<String> ancestorPath = new ArrayList<String>();
+ Path pathElement=path;
+ while(pathElement != null){
+ ancestorPath.add(0, pathElement.getName());
+ pathElement = pathElement.getParent();
+ }
+ ancestorPath.remove(0);
+ for(String s : ancestorPath){
+ absolutePath.append("/");
+ absolutePath.append(s);
+ }
+ return new String(absolutePath);
+ }
+
+ private static VersionedProtocol getProtocol(Class protocolClass, InetSocketAddress inetAddress, JobConf jobConf) throws IOException{
+ VersionedProtocol versionedProtocol = RPC.getProxy(protocolClass, ClientProtocol.versionID, inetAddress, jobConf);
+ return versionedProtocol;
+ }
+
+ private static List<HadoopFileSplit> getFileBlocks(String absolutePath,FileStatus fileStatus){
+ List<HadoopFileSplit> hadoopFileSplits = new ArrayList<HadoopFileSplit>();
+ try{
+ LocatedBlocks locatedBlocks = namenode.getBlockLocations(absolutePath, 0, fileStatus.getLen());
+ long blockSize = fileSystem.getBlockSize(new Path(absolutePath));
+ if(locatedBlocks !=null){
+ int index = 0;
+ for(LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()){
+ DatanodeInfo[] datanodeInfos = locatedBlock.getLocations(); // all datanodes having this block
+ String [] hostnames = new String[datanodeInfos.length];
+ int datanodeCount =0;
+ for(DatanodeInfo datanodeInfo : datanodeInfos){
+ hostnames[datanodeCount++] = datanodeInfo.getHostName();
+ }
+ HadoopFileSplit hadoopFileSplit = new HadoopFileSplit(absolutePath,new Long(index * blockSize).longValue(),new Long(blockSize).longValue(),hostnames);
+ hadoopFileSplits.add(hadoopFileSplit);
+ index++;
+ }
+ }
+ }catch(Exception e){
+ e.printStackTrace();
+ }
+ return hadoopFileSplits;
+ }
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/HadoopFileSplit.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/HadoopFileSplit.java
new file mode 100644
index 0000000..6f3810c
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/HadoopFileSplit.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.hadoop.util;
+
+import java.io.Serializable;
+
+import org.apache.hadoop.fs.Path;
+
+public class HadoopFileSplit implements Serializable{
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+ private String filePath;
+ private long start;
+ private long length;
+ private String[] hosts;
+
+ public HadoopFileSplit(String filePath, long start, long length, String[] hosts){
+ this.filePath = filePath;
+ this.start = start;
+ this.length = length;
+ this.hosts = hosts;
+ }
+
+ public String getFile() {
+ return filePath;
+ }
+
+ public void setFile(String file) {
+ this.filePath = file;
+ }
+
+ public long getStart() {
+ return start;
+ }
+
+ public void setStart(long start) {
+ this.start = start;
+ }
+
+ public long getLength() {
+ return length;
+ }
+
+ public void setLength(long length) {
+ this.length = length;
+ }
+
+ public String[] getHosts() {
+ return hosts;
+ }
+
+ public void setHosts(String[] hosts) {
+ this.hosts = hosts;
+ }
+
+ public String toString(){
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append(filePath + " " + start + " " + length + "\n");
+ for(String host : hosts){
+ stringBuilder.append(host);
+ stringBuilder.append(",");
+ }
+ return new String(stringBuilder);
+ }
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/IHadoopClassFactory.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/IHadoopClassFactory.java
new file mode 100644
index 0000000..c4f1d3c
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/IHadoopClassFactory.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.hadoop.util;
+
+import java.io.Serializable;
+
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.Reducer;
+
+public interface IHadoopClassFactory extends Serializable{
+
+ public Mapper createMapper(String mapClassName) throws Exception;
+
+ public Reducer createReducer(String reduceClassName) throws Exception;
+
+ public Class loadClass(String className) throws Exception;
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/PreappendLongWritableMapper.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/PreappendLongWritableMapper.java
new file mode 100644
index 0000000..e561522
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/hadoop/util/PreappendLongWritableMapper.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.hadoop.util;
+
+import java.util.Properties;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+
+import edu.uci.ics.hyracks.api.dataflow.IDataWriter;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.coreops.IMapper;
+
+public class PreappendLongWritableMapper implements IMapper {
+
+ @Override
+ public void map(Object[] data, IDataWriter<Object[]> writer) throws HyracksDataException {
+ writer.writeData(new Object[] { new LongWritable(0), new Text(String.valueOf(data[0])) });
+ }
+}
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/JobControl.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/JobControl.java
new file mode 100644
index 0000000..3259649
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/JobControl.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.job;
+
+import java.rmi.RemoteException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.api.job.statistics.JobStatistics;
+import edu.uci.ics.hyracks.api.job.statistics.StageStatistics;
+import edu.uci.ics.hyracks.api.job.statistics.StageletStatistics;
+
+public class JobControl {
+ private static final long serialVersionUID = 1L;
+
+ private final JobManager jobManager;
+
+ private final JobPlan jobPlan;
+
+ private final UUID jobId;
+
+ private final Map<UUID, StageProgress> stageProgressMap;
+
+ private final Set<UUID> completeStages;
+
+ private JobStatus jobStatus;
+
+ private JobStatistics jobStatistics;
+
+ public JobControl(JobManager jobManager, JobPlan jobPlan) throws RemoteException {
+ this.jobManager = jobManager;
+ this.jobPlan = jobPlan;
+ jobId = UUID.randomUUID();
+ stageProgressMap = new HashMap<UUID, StageProgress>();
+ completeStages = new HashSet<UUID>();
+ jobStatus = JobStatus.INITIALIZED;
+ jobStatistics = new JobStatistics();
+ }
+
+ public JobPlan getJobPlan() {
+ return jobPlan;
+ }
+
+ public UUID getJobId() {
+ return jobId;
+ }
+
+ public synchronized JobStatus getJobStatus() {
+ return jobStatus;
+ }
+
+ public Set<UUID> getCompletedStages() {
+ return completeStages;
+ }
+
+ public void setStatus(JobStatus status) {
+ jobStatus = status;
+ }
+
+ public StageProgress getStageProgress(int stageId) {
+ return stageProgressMap.get(stageId);
+ }
+
+ public void setStageProgress(UUID stageId, StageProgress stageProgress) {
+ stageProgressMap.put(stageId, stageProgress);
+ }
+
+ public synchronized void notifyStageletComplete(UUID stageId, String nodeId, StageletStatistics ss)
+ throws Exception {
+ StageProgress stageProgress = stageProgressMap.get(stageId);
+ stageProgress.markNodeComplete(nodeId);
+ StageStatistics stageStatistics = stageProgress.getStageStatistics();
+ stageStatistics.addStageletStatistics(ss);
+ if (stageProgress.stageComplete()) {
+ jobStatistics.addStageStatistics(stageStatistics);
+ stageProgressMap.remove(stageId);
+ completeStages.add(stageId);
+ jobManager.advanceJob(this);
+ }
+ }
+
+ public synchronized JobStatistics waitForCompletion() throws Exception {
+ while (jobStatus != JobStatus.TERMINATED) {
+ wait();
+ }
+ return jobStatistics;
+ }
+
+ public synchronized void notifyJobComplete() {
+ jobStatus = JobStatus.TERMINATED;
+ notifyAll();
+ }
+
+ public JobStatistics getJobStatistics() {
+ return jobStatistics;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/JobManager.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/JobManager.java
new file mode 100644
index 0000000..f976bb9
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/JobManager.java
@@ -0,0 +1,352 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.job;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.Vector;
+import java.util.concurrent.Semaphore;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.comm.Endpoint;
+import edu.uci.ics.hyracks.api.constraints.AbsoluteLocationConstraint;
+import edu.uci.ics.hyracks.api.constraints.LocationConstraint;
+import edu.uci.ics.hyracks.api.constraints.PartitionConstraint;
+import edu.uci.ics.hyracks.api.controller.INodeController;
+import edu.uci.ics.hyracks.api.dataflow.ActivityNodeId;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.PortInstanceId;
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.JobStage;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.api.job.statistics.JobStatistics;
+import edu.uci.ics.hyracks.api.job.statistics.StageletStatistics;
+import edu.uci.ics.hyracks.controller.ClusterControllerService;
+import edu.uci.ics.hyracks.dataflow.base.IOperatorDescriptorVisitor;
+import edu.uci.ics.hyracks.dataflow.util.PlanUtils;
+
+public class JobManager {
+ private static final Logger LOGGER = Logger.getLogger(JobManager.class.getName());
+ private ClusterControllerService ccs;
+
+ private final Map<UUID, JobControl> jobMap;
+
+ public JobManager(ClusterControllerService ccs) {
+ this.ccs = ccs;
+ jobMap = new HashMap<UUID, JobControl>();
+ }
+
+ public synchronized UUID createJob(JobSpecification jobSpec, EnumSet<JobFlag> jobFlags) throws Exception {
+ JobPlanBuilder builder = new JobPlanBuilder();
+ builder.init(jobSpec, jobFlags);
+ JobControl jc = new JobControl(this, builder.plan());
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(jc.getJobPlan().toString());
+ }
+ jobMap.put(jc.getJobId(), jc);
+
+ return jc.getJobId();
+ }
+
+ public synchronized void start(UUID jobId) throws Exception {
+ JobControl jobControlImpl = jobMap.get(jobId);
+ LOGGER
+ .info("Starting job: " + jobControlImpl.getJobId() + ", Current status: "
+ + jobControlImpl.getJobStatus());
+ if (jobControlImpl.getJobStatus() != JobStatus.INITIALIZED) {
+ return;
+ }
+ jobControlImpl.getJobStatistics().setStartTime(new Date());
+ jobControlImpl.setStatus(JobStatus.RUNNING);
+ schedule(jobControlImpl);
+ }
+
+ public synchronized void advanceJob(JobControl jobControlImpl) throws Exception {
+ schedule(jobControlImpl);
+ }
+
+ private void schedule(JobControl jobControlImpl) throws Exception {
+ JobPlan plan = jobControlImpl.getJobPlan();
+ JobStage endStage = plan.getEndStage();
+
+ Set<UUID> completedStages = jobControlImpl.getCompletedStages();
+ List<JobStage> runnableStages = new ArrayList<JobStage>();
+ findRunnableStages(endStage, runnableStages, completedStages, new HashSet<UUID>());
+ if (runnableStages.size() == 1 && runnableStages.get(0).getTasks().isEmpty()) {
+ LOGGER.info("Job " + jobControlImpl.getJobId() + " complete");
+ jobControlImpl.getJobStatistics().setEndTime(new Date());
+ cleanUp(jobControlImpl);
+ jobControlImpl.notifyJobComplete();
+ } else {
+ for (JobStage s : runnableStages) {
+ if (s.isStarted()) {
+ continue;
+ }
+ startStage(jobControlImpl, s);
+ }
+ }
+ }
+
+ private void cleanUp(JobControl jc) {
+ jobMap.remove(jc.getJobId());
+ ccs.notifyJobComplete(jc.getJobId());
+ }
+
+ private void startStage(JobControl jc, JobStage stage) throws Exception {
+ stage.setStarted();
+ Set<String> candidateNodes = deploy(jc, stage);
+ for (String nodeId : candidateNodes) {
+ ccs.lookupNode(nodeId).startStage(jc.getJobId(), stage.getId());
+ }
+ }
+
+ private void findRunnableStages(JobStage s, List<JobStage> runnableStages, Set<UUID> completedStages, Set<UUID> seen) {
+ boolean runnable = true;
+ if (seen.contains(s.getId())) {
+ return;
+ }
+ seen.add(s.getId());
+ for (JobStage dep : s.getDependencies()) {
+ boolean depComplete = completedStages.contains(dep.getId());
+ runnable = runnable && depComplete;
+ if (!depComplete) {
+ findRunnableStages(dep, runnableStages, completedStages, seen);
+ }
+ }
+ if (runnable) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Runnable stage: " + s);
+ }
+ runnableStages.add(s);
+ }
+ }
+
+ private Set<String> deploy(JobControl jc, JobStage stage) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Deploying: " + stage);
+ }
+ Set<String> candidateNodes = plan(jc, stage);
+ StageProgress stageProgress = new StageProgress(stage.getId());
+ stageProgress.addPendingNodes(candidateNodes);
+ Map<PortInstanceId, Endpoint> globalPortMap = runRemote(new Phase1Installer(jc, stage),
+ new PortMapMergingAccumulator(), candidateNodes);
+ runRemote(new Phase2Installer(jc, stage, globalPortMap), null, candidateNodes);
+ runRemote(new Phase3Installer(jc, stage), null, candidateNodes);
+ jc.setStageProgress(stage.getId(), stageProgress);
+ return candidateNodes;
+ }
+
+ private interface RemoteOp<T> {
+ public T execute(INodeController node) throws Exception;
+ }
+
+ private interface Accumulator<T, R> {
+ public void accumulate(T o);
+
+ public R getResult();
+ }
+
+ private static class Phase1Installer implements RemoteOp<Map<PortInstanceId, Endpoint>> {
+ private JobControl jc;
+ private JobStage stage;
+
+ public Phase1Installer(JobControl jc, JobStage stage) {
+ this.jc = jc;
+ this.stage = stage;
+ }
+
+ @Override
+ public Map<PortInstanceId, Endpoint> execute(INodeController node) throws Exception {
+ return node.initializeJobletPhase1(jc.getJobId(), jc.getJobPlan(), stage);
+ }
+
+ @Override
+ public String toString() {
+ return jc.getJobId() + " Distribution Phase 1";
+ }
+ }
+
+ private static class Phase2Installer implements RemoteOp<Void> {
+ private JobControl jc;
+ private JobStage stage;
+ private Map<PortInstanceId, Endpoint> globalPortMap;
+
+ public Phase2Installer(JobControl jc, JobStage stage, Map<PortInstanceId, Endpoint> globalPortMap) {
+ this.jc = jc;
+ this.stage = stage;
+ this.globalPortMap = globalPortMap;
+ }
+
+ @Override
+ public Void execute(INodeController node) throws Exception {
+ node.initializeJobletPhase2(jc.getJobId(), jc.getJobPlan(), stage, globalPortMap);
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return jc.getJobId() + " Distribution Phase 2";
+ }
+ }
+
+ private static class Phase3Installer implements RemoteOp<Void> {
+ private JobControl jc;
+ private JobStage stage;
+
+ public Phase3Installer(JobControl jc, JobStage stage) {
+ this.jc = jc;
+ this.stage = stage;
+ }
+
+ @Override
+ public Void execute(INodeController node) throws Exception {
+ node.commitJobletInitialization(jc.getJobId(), jc.getJobPlan(), stage);
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return jc.getJobId() + " Distribution Phase 3";
+ }
+ }
+
+ private static class PortMapMergingAccumulator implements
+ Accumulator<Map<PortInstanceId, Endpoint>, Map<PortInstanceId, Endpoint>> {
+ Map<PortInstanceId, Endpoint> portMap = new HashMap<PortInstanceId, Endpoint>();
+
+ @Override
+ public void accumulate(Map<PortInstanceId, Endpoint> o) {
+ portMap.putAll(o);
+ }
+
+ @Override
+ public Map<PortInstanceId, Endpoint> getResult() {
+ return portMap;
+ }
+ }
+
+ private <T, R> R runRemote(final RemoteOp<T> remoteOp, final Accumulator<T, R> accumulator,
+ Set<String> candidateNodes) throws Exception {
+ LOGGER.log(Level.INFO, remoteOp + " : " + candidateNodes);
+
+ final Semaphore installComplete = new Semaphore(candidateNodes.size());
+ final List<Exception> errors = new Vector<Exception>();
+ for (final String nodeId : candidateNodes) {
+ final INodeController node = ccs.lookupNode(nodeId);
+
+ installComplete.acquire();
+ Runnable remoteRunner = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ T t = remoteOp.execute(node);
+ if (accumulator != null) {
+ synchronized (accumulator) {
+ accumulator.accumulate(t);
+ }
+ }
+ } catch (Exception e) {
+ errors.add(e);
+ } finally {
+ installComplete.release();
+ }
+ }
+ };
+
+ ccs.getExecutor().execute(remoteRunner);
+ }
+ installComplete.acquire(candidateNodes.size());
+ if (!errors.isEmpty()) {
+ throw errors.get(0);
+ }
+ return accumulator == null ? null : accumulator.getResult();
+ }
+
+ private Set<String> plan(JobControl jc, JobStage stage) throws Exception {
+ LOGGER.log(Level.INFO, String.valueOf(jc.getJobId()) + ": Planning");
+
+ final Set<OperatorDescriptorId> opSet = new HashSet<OperatorDescriptorId>();
+ for (ActivityNodeId t : stage.getTasks()) {
+ opSet.add(jc.getJobPlan().getActivityNodeMap().get(t).getOwner().getOperatorId());
+ }
+
+ final Set<String> candidateNodes = new HashSet<String>();
+
+ IOperatorDescriptorVisitor visitor = new IOperatorDescriptorVisitor() {
+ @Override
+ public void visit(IOperatorDescriptor op) throws Exception {
+ if (!opSet.contains(op.getOperatorId())) {
+ return;
+ }
+ String[] partitions = op.getPartitions();
+ if (partitions == null) {
+ PartitionConstraint pc = op.getPartitionConstraint();
+ LocationConstraint[] lcs = pc.getLocationConstraints();
+ String[] assignment = new String[lcs.length];
+ for (int i = 0; i < lcs.length; ++i) {
+ String nodeId = ((AbsoluteLocationConstraint) lcs[i]).getLocationId();
+ assignment[i] = nodeId;
+ }
+ op.setPartitions(assignment);
+ partitions = assignment;
+ }
+ for (String p : partitions) {
+ candidateNodes.add(p);
+ }
+ }
+ };
+
+ PlanUtils.visit(jc.getJobPlan().getJobSpecification(), visitor);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(stage + " Candidate nodes: " + candidateNodes);
+ }
+ return candidateNodes;
+ }
+
+ public synchronized void notifyStageletComplete(UUID jobId, UUID stageId, String nodeId,
+ StageletStatistics statistics) throws Exception {
+ JobControl jc = jobMap.get(jobId);
+ if (jc != null) {
+ jc.notifyStageletComplete(stageId, nodeId, statistics);
+ }
+ }
+
+ public synchronized JobStatus getJobStatus(UUID jobId) {
+ JobControl jc = jobMap.get(jobId);
+ return jc.getJobStatus();
+ }
+
+ public JobStatistics waitForCompletion(UUID jobId) throws Exception {
+ JobControl jc;
+ synchronized (this) {
+ jc = jobMap.get(jobId);
+ }
+ if (jc != null) {
+ return jc.waitForCompletion();
+ }
+ return null;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/JobPlanBuilder.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/JobPlanBuilder.java
new file mode 100644
index 0000000..61cf70f
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/JobPlanBuilder.java
@@ -0,0 +1,235 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.job;
+
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.dataflow.ActivityNodeId;
+import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder;
+import edu.uci.ics.hyracks.api.dataflow.IActivityNode;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobPlan;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.JobStage;
+import edu.uci.ics.hyracks.api.util.Pair;
+import edu.uci.ics.hyracks.dataflow.base.IOperatorDescriptorVisitor;
+import edu.uci.ics.hyracks.dataflow.util.PlanUtils;
+
+public class JobPlanBuilder implements IActivityGraphBuilder {
+ private static final Logger LOGGER = Logger.getLogger(JobPlanBuilder.class.getName());
+
+ private JobPlan plan;
+
+ @Override
+ public void addBlockingEdge(IActivityNode blocker, IActivityNode blocked) {
+ addToValueSet(plan.getBlocker2BlockedMap(), blocker.getActivityNodeId(), blocked.getActivityNodeId());
+ addToValueSet(plan.getBlocked2BlockerMap(), blocked.getActivityNodeId(), blocker.getActivityNodeId());
+ }
+
+ @Override
+ public void addSourceEdge(int operatorInputIndex, IActivityNode task, int taskInputIndex) {
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest("Adding source edge: " + task.getOwner().getOperatorId() + ":" + operatorInputIndex + " -> "
+ + task.getActivityNodeId() + ":" + taskInputIndex);
+ }
+ insertIntoIndexedMap(plan.getTaskInputMap(), task.getActivityNodeId(), taskInputIndex, operatorInputIndex);
+ insertIntoIndexedMap(plan.getOperatorInputMap(), task.getOwner().getOperatorId(), operatorInputIndex, task
+ .getActivityNodeId());
+ }
+
+ @Override
+ public void addTargetEdge(int operatorOutputIndex, IActivityNode task, int taskOutputIndex) {
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest("Adding target edge: " + task.getOwner().getOperatorId() + ":" + operatorOutputIndex + " -> "
+ + task.getActivityNodeId() + ":" + taskOutputIndex);
+ }
+ insertIntoIndexedMap(plan.getTaskOutputMap(), task.getActivityNodeId(), taskOutputIndex, operatorOutputIndex);
+ insertIntoIndexedMap(plan.getOperatorOutputMap(), task.getOwner().getOperatorId(), operatorOutputIndex, task
+ .getActivityNodeId());
+ }
+
+ @Override
+ public void addTask(IActivityNode task) {
+ plan.getActivityNodeMap().put(task.getActivityNodeId(), task);
+ addToValueSet(plan.getOperatorTaskMap(), task.getOwner().getOperatorId(), task.getActivityNodeId());
+ }
+
+ private <K, V> void addToValueSet(Map<K, Set<V>> map, K n1, V n2) {
+ Set<V> targets = map.get(n1);
+ if (targets == null) {
+ targets = new HashSet<V>();
+ map.put(n1, targets);
+ }
+ targets.add(n2);
+ }
+
+ private <T> void extend(List<T> list, int index) {
+ int n = list.size();
+ for (int i = n; i <= index; ++i) {
+ list.add(null);
+ }
+ }
+
+ private Pair<ActivityNodeId, ActivityNodeId> findMergePair(JobPlan plan, JobSpecification spec, Set<JobStage> eqSets) {
+ Map<ActivityNodeId, IActivityNode> activityNodeMap = plan.getActivityNodeMap();
+ for (JobStage eqSet : eqSets) {
+ for (ActivityNodeId t : eqSet.getTasks()) {
+ IOperatorDescriptor owner = activityNodeMap.get(t).getOwner();
+ List<Integer> inputList = plan.getTaskInputMap().get(t);
+ if (inputList != null) {
+ for (Integer idx : inputList) {
+ IConnectorDescriptor conn = spec.getInputConnectorDescriptor(owner, idx);
+ OperatorDescriptorId producerId = spec.getProducer(conn).getOperatorId();
+ int producerOutputIndex = spec.getProducerOutputIndex(conn);
+ ActivityNodeId inTask = plan.getOperatorOutputMap().get(producerId).get(producerOutputIndex);
+ if (!eqSet.getTasks().contains(inTask)) {
+ return new Pair<ActivityNodeId, ActivityNodeId>(t, inTask);
+ }
+ }
+ }
+ List<Integer> outputList = plan.getTaskOutputMap().get(t);
+ if (outputList != null) {
+ for (Integer idx : outputList) {
+ IConnectorDescriptor conn = spec.getOutputConnectorDescriptor(owner, idx);
+ OperatorDescriptorId consumerId = spec.getConsumer(conn).getOperatorId();
+ int consumerInputIndex = spec.getConsumerInputIndex(conn);
+ ActivityNodeId outTask = plan.getOperatorInputMap().get(consumerId).get(consumerInputIndex);
+ if (!eqSet.getTasks().contains(outTask)) {
+ return new Pair<ActivityNodeId, ActivityNodeId>(t, outTask);
+ }
+ }
+ }
+ }
+ }
+ return null;
+ }
+
+ private JobStage inferStages() throws Exception {
+ JobSpecification spec = plan.getJobSpecification();
+
+ /*
+ * Build initial equivalence sets map. We create a map such that for
+ * each IOperatorTask, t -> { t }
+ */
+ Map<ActivityNodeId, JobStage> stageMap = new HashMap<ActivityNodeId, JobStage>();
+ Set<JobStage> stages = new HashSet<JobStage>();
+ for (Set<ActivityNodeId> taskIds : plan.getOperatorTaskMap().values()) {
+ for (ActivityNodeId taskId : taskIds) {
+ Set<ActivityNodeId> eqSet = new HashSet<ActivityNodeId>();
+ eqSet.add(taskId);
+ JobStage stage = new JobStage(eqSet);
+ stageMap.put(taskId, stage);
+ stages.add(stage);
+ }
+ }
+
+ boolean changed = true;
+ while (changed) {
+ changed = false;
+ Pair<ActivityNodeId, ActivityNodeId> pair = findMergePair(plan, spec, stages);
+ if (pair != null) {
+ merge(stageMap, stages, pair.first, pair.second);
+ changed = true;
+ }
+ }
+
+ JobStage endStage = new JobStage(new HashSet<ActivityNodeId>());
+ Map<ActivityNodeId, Set<ActivityNodeId>> blocker2BlockedMap = plan.getBlocker2BlockedMap();
+ for (JobStage s : stages) {
+ endStage.addDependency(s);
+ s.addDependent(endStage);
+ Set<JobStage> blockedStages = new HashSet<JobStage>();
+ for (ActivityNodeId t : s.getTasks()) {
+ Set<ActivityNodeId> blockedTasks = blocker2BlockedMap.get(t);
+ if (blockedTasks != null) {
+ for (ActivityNodeId bt : blockedTasks) {
+ blockedStages.add(stageMap.get(bt));
+ }
+ }
+ }
+ for (JobStage bs : blockedStages) {
+ bs.addDependency(s);
+ s.addDependent(bs);
+ }
+ }
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inferred " + (stages.size() + 1) + " stages");
+ for (JobStage s : stages) {
+ LOGGER.info(s.toString());
+ }
+ LOGGER.info("SID: ENDSTAGE");
+ }
+ return endStage;
+ }
+
+ public void init(JobSpecification jobSpec, EnumSet<JobFlag> jobFlags) {
+ plan = new JobPlan(jobSpec, jobFlags);
+ }
+
+ private <K, V> void insertIntoIndexedMap(Map<K, List<V>> map, K key, int index, V value) {
+ List<V> vList = map.get(key);
+ if (vList == null) {
+ vList = new ArrayList<V>();
+ map.put(key, vList);
+ }
+ extend(vList, index);
+ vList.set(index, value);
+ }
+
+ private void merge(Map<ActivityNodeId, JobStage> eqSetMap, Set<JobStage> eqSets, ActivityNodeId t1,
+ ActivityNodeId t2) {
+ JobStage stage1 = eqSetMap.get(t1);
+ Set<ActivityNodeId> s1 = stage1.getTasks();
+ JobStage stage2 = eqSetMap.get(t2);
+ Set<ActivityNodeId> s2 = stage2.getTasks();
+
+ Set<ActivityNodeId> mergedSet = new HashSet<ActivityNodeId>();
+ mergedSet.addAll(s1);
+ mergedSet.addAll(s2);
+
+ eqSets.remove(stage1);
+ eqSets.remove(stage2);
+ JobStage mergedStage = new JobStage(mergedSet);
+ eqSets.add(mergedStage);
+
+ for (ActivityNodeId t : mergedSet) {
+ eqSetMap.put(t, mergedStage);
+ }
+ }
+
+ public JobPlan plan() throws Exception {
+ PlanUtils.visit(plan.getJobSpecification(), new IOperatorDescriptorVisitor() {
+ @Override
+ public void visit(IOperatorDescriptor op) throws Exception {
+ op.contributeTaskGraph(JobPlanBuilder.this);
+ }
+ });
+ JobStage endStage = inferStages();
+ plan.setEndStage(endStage);
+
+ return plan;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/Joblet.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/Joblet.java
new file mode 100644
index 0000000..5f38e56
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/Joblet.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.job;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.Executor;
+
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.statistics.StageletStatistics;
+import edu.uci.ics.hyracks.controller.NodeControllerService;
+
+public class Joblet {
+ private static final long serialVersionUID = 1L;
+
+ private final NodeControllerService nodeController;
+
+ private final UUID jobId;
+
+ private final Map<UUID, Stagelet> stageletMap;
+
+ private final Map<OperatorDescriptorId, Map<Integer, IOperatorEnvironment>> envMap;
+
+ public Joblet(NodeControllerService nodeController, UUID jobId) throws Exception {
+ this.nodeController = nodeController;
+ this.jobId = jobId;
+ stageletMap = new HashMap<UUID, Stagelet>();
+ envMap = new HashMap<OperatorDescriptorId, Map<Integer, IOperatorEnvironment>>();
+ }
+
+ public UUID getJobId() {
+ return jobId;
+ }
+
+ public IOperatorEnvironment getEnvironment(IOperatorDescriptor hod, int partition) {
+ if (!envMap.containsKey(hod.getOperatorId())) {
+ envMap.put(hod.getOperatorId(), new HashMap<Integer, IOperatorEnvironment>());
+ }
+ Map<Integer, IOperatorEnvironment> opEnvMap = envMap.get(hod.getOperatorId());
+ if (!opEnvMap.containsKey(partition)) {
+ opEnvMap.put(partition, new OperatorEnvironmentImpl());
+ }
+ return opEnvMap.get(partition);
+ }
+
+ private static final class OperatorEnvironmentImpl implements IOperatorEnvironment {
+ private final Map<String, Object> map;
+
+ public OperatorEnvironmentImpl() {
+ map = new HashMap<String, Object>();
+ }
+
+ @Override
+ public Object get(String name) {
+ return map.get(name);
+ }
+
+ @Override
+ public void set(String name, Object value) {
+ map.put(name, value);
+ }
+ }
+
+ public void setStagelet(UUID stageId, Stagelet stagelet) {
+ stageletMap.put(stageId, stagelet);
+ }
+
+ public Stagelet getStagelet(UUID stageId) throws Exception {
+ return stageletMap.get(stageId);
+ }
+
+ public Executor getExecutor() {
+ return nodeController.getExecutor();
+ }
+
+ public synchronized void notifyStageletComplete(UUID stageId, StageletStatistics stats) throws Exception {
+ stageletMap.remove(stageId);
+ nodeController.notifyStageComplete(jobId, stageId, stats);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/StageProgress.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/StageProgress.java
new file mode 100644
index 0000000..3357012
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/StageProgress.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.job;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.UUID;
+
+import edu.uci.ics.hyracks.api.job.statistics.StageStatistics;
+
+public class StageProgress {
+ private final UUID stageId;
+
+ private final Set<String> pendingNodes;
+
+ private final StageStatistics stageStatistics;
+
+ public StageProgress(UUID stageId) {
+ this.stageId = stageId;
+ pendingNodes = new HashSet<String>();
+ stageStatistics = new StageStatistics();
+ stageStatistics.setStageId(stageId);
+ }
+
+ public UUID getStageId() {
+ return stageId;
+ }
+
+ public void addPendingNodes(Set<String> nodes) {
+ pendingNodes.addAll(nodes);
+ }
+
+ public void markNodeComplete(String nodeId) {
+ pendingNodes.remove(nodeId);
+ }
+
+ public boolean stageComplete() {
+ return pendingNodes.isEmpty();
+ }
+
+ public StageStatistics getStageStatistics() {
+ return stageStatistics;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/Stagelet.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/Stagelet.java
new file mode 100644
index 0000000..e82fa5b
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/job/Stagelet.java
@@ -0,0 +1,136 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.job;
+
+import java.rmi.RemoteException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.comm.Endpoint;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.OperatorInstanceId;
+import edu.uci.ics.hyracks.api.job.statistics.StageletStatistics;
+import edu.uci.ics.hyracks.runtime.OperatorRunnable;
+
+public class Stagelet {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOGGER = Logger.getLogger(Stagelet.class.getName());
+
+ private final Joblet joblet;
+
+ private final UUID stageId;
+
+ private final Map<OperatorInstanceId, OperatorRunnable> honMap;
+
+ private List<Endpoint> endpointList;
+
+ private boolean started;
+
+ private final Set<OperatorInstanceId> pendingOperators;
+
+ private final StageletStatistics stats;
+
+ public Stagelet(Joblet joblet, UUID stageId, String nodeId) throws RemoteException {
+ this.joblet = joblet;
+ this.stageId = stageId;
+ pendingOperators = new HashSet<OperatorInstanceId>();
+ started = false;
+ honMap = new HashMap<OperatorInstanceId, OperatorRunnable>();
+ stats = new StageletStatistics();
+ stats.setNodeId(nodeId);
+ }
+
+ public void setOperator(OperatorDescriptorId odId, int partition, OperatorRunnable hon) {
+ honMap.put(new OperatorInstanceId(odId, partition), hon);
+ }
+
+ public Map<OperatorInstanceId, OperatorRunnable> getOperatorMap() {
+ return honMap;
+ }
+
+ public void setEndpointList(List<Endpoint> endpointList) {
+ this.endpointList = endpointList;
+ }
+
+ public List<Endpoint> getEndpointList() {
+ return endpointList;
+ }
+
+ public synchronized void start() throws Exception {
+ if (started) {
+ throw new Exception("Joblet already started");
+ }
+ started = true;
+ stats.setStartTime(new Date());
+ notifyAll();
+ }
+
+ public void installRunnable(final OperatorInstanceId opIId) {
+ pendingOperators.add(opIId);
+ final OperatorRunnable hon = honMap.get(opIId);
+ joblet.getExecutor().execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ waitUntilStarted();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ return;
+ }
+ try {
+ LOGGER.log(Level.INFO, "Starting runnable for operator: " + joblet.getJobId() + ":" + stageId + ":"
+ + opIId.getOperatorId() + ":" + opIId.getPartition());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ try {
+ hon.run();
+ } finally {
+ notifyOperatorCompletion(opIId);
+ }
+ }
+ });
+ }
+
+ protected synchronized void notifyOperatorCompletion(OperatorInstanceId opIId) {
+ pendingOperators.remove(opIId);
+ if (pendingOperators.isEmpty()) {
+ stats.setEndTime(new Date());
+ try {
+ joblet.notifyStageletComplete(stageId, stats);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ private synchronized void waitUntilStarted() throws InterruptedException {
+ while (!started) {
+ wait();
+ }
+ }
+
+ public StageletStatistics getStatistics() {
+ return stats;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/resources/ResourceManager.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/resources/ResourceManager.java
new file mode 100644
index 0000000..38be3ff
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/resources/ResourceManager.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.resources;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public final class ResourceManager {
+ private final HyracksContext ctx;
+
+ public ResourceManager(HyracksContext ctx) {
+ this.ctx = ctx;
+ }
+
+ public ByteBuffer allocateFrame() {
+ return ByteBuffer.allocate(ctx.getFrameSize());
+ }
+
+ public File createFile(String prefix, String suffix) throws IOException {
+ return File.createTempFile(prefix, suffix);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/runtime/OperatorRunnable.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/runtime/OperatorRunnable.java
new file mode 100644
index 0000000..5964d07
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/runtime/OperatorRunnable.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.runtime;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.context.HyracksContext;
+
+public class OperatorRunnable implements Runnable {
+ private IOperatorNodePushable opNode;
+ private IFrameReader reader;
+ private ByteBuffer buffer;
+
+ public OperatorRunnable(HyracksContext ctx, IOperatorNodePushable opNode) {
+ this.opNode = opNode;
+ buffer = ctx.getResourceManager().allocateFrame();
+ }
+
+ public void setFrameWriter(int index, IFrameWriter writer) {
+ opNode.setFrameWriter(index, writer);
+ }
+
+ public void setFrameReader(IFrameReader reader) {
+ this.reader = reader;
+ }
+
+ @Override
+ public void run() {
+ try {
+ opNode.open();
+ if (reader != null) {
+ reader.open();
+ while (reader.nextFrame(buffer)) {
+ buffer.flip();
+ opNode.nextFrame(buffer);
+ buffer.compact();
+ }
+ reader.close();
+ }
+ opNode.close();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/service/AbstractService.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/service/AbstractService.java
new file mode 100644
index 0000000..fcb6ea4
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/service/AbstractService.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.service;
+
+public abstract class AbstractService implements IService {
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/service/IService.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/service/IService.java
new file mode 100644
index 0000000..edfdc83
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/service/IService.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.service;
+
+import edu.uci.ics.hyracks.util.ILifeCycle;
+
+public interface IService extends ILifeCycle {
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/util/ILifeCycle.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/util/ILifeCycle.java
new file mode 100644
index 0000000..e526fa6
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/util/ILifeCycle.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.util;
+
+public interface ILifeCycle {
+ public void start() throws Exception;
+
+ public void stop() throws Exception;
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/util/PropertiesAccess.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/util/PropertiesAccess.java
new file mode 100644
index 0000000..3e01dc9
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/util/PropertiesAccess.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.util;
+
+import java.util.Properties;
+
+public class PropertiesAccess {
+ public static boolean safeBoolean(Properties props, String pName, boolean defaultValue) {
+ String pValue = props.getProperty(pName);
+ if (pValue == null) {
+ return defaultValue;
+ }
+ try {
+ return Boolean.parseBoolean(pValue);
+ } catch (Exception e) {
+ return defaultValue;
+ }
+ }
+
+ public static String safeString(Properties props, String pName, String defaultValue) {
+ String pValue = props.getProperty(pName);
+ if (pValue == null) {
+ return defaultValue;
+ }
+ return pValue;
+ }
+
+ public static int safeInt(Properties props, String pName, int defaultValue) {
+ String pValue = props.getProperty(pName);
+ if (pValue == null) {
+ return defaultValue;
+ }
+ try {
+ return Integer.parseInt(pValue);
+ } catch (Exception e) {
+ return defaultValue;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/util/ReflectionUtils.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/util/ReflectionUtils.java
new file mode 100644
index 0000000..bb391d7
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/util/ReflectionUtils.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.util;
+
+public class ReflectionUtils {
+ public static <T> T createInstance(Class<? extends T> klass) {
+ T instance = null;
+ try {
+ instance = klass.newInstance();
+ } catch (InstantiationException e) {
+ throw new RuntimeException(e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ return instance;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/java/edu/uci/ics/hyracks/web/WebServer.java b/hyracks-core/src/main/java/edu/uci/ics/hyracks/web/WebServer.java
new file mode 100644
index 0000000..b6ab52b
--- /dev/null
+++ b/hyracks-core/src/main/java/edu/uci/ics/hyracks/web/WebServer.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.web;
+
+import org.eclipse.jetty.server.Connector;
+import org.eclipse.jetty.server.Handler;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.handler.ContextHandlerCollection;
+import org.eclipse.jetty.server.nio.SelectChannelConnector;
+
+public class WebServer {
+ private Server server;
+ private SelectChannelConnector connector;
+
+ public WebServer(Handler[] handlers) throws Exception {
+ server = new Server();
+
+ connector = new SelectChannelConnector();
+
+ server.setConnectors(new Connector[] { connector });
+
+ ContextHandlerCollection handler = new ContextHandlerCollection();
+ handler.setHandlers(handlers);
+ server.setHandler(handler);
+ }
+
+ public void setPort(int port) {
+ connector.setPort(port);
+ }
+
+ public void start() throws Exception {
+ server.start();
+ }
+
+ public void stop() throws Exception {
+ server.stop();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/main/scripts/run.cmd b/hyracks-core/src/main/scripts/run.cmd
new file mode 100644
index 0000000..2df8467
--- /dev/null
+++ b/hyracks-core/src/main/scripts/run.cmd
@@ -0,0 +1,62 @@
+@ECHO OFF
+SETLOCAL
+
+:: Licensed to the Apache Software Foundation (ASF) under one or more
+:: contributor license agreements. See the NOTICE file distributed with
+:: this work for additional information regarding copyright ownership.
+:: The ASF licenses this file to You under the Apache License, Version 2.0
+:: (the "License"); you may not use this file except in compliance with
+:: the License. You may obtain a copy of the License at
+::
+:: http://www.apache.org/licenses/LICENSE-2.0
+::
+:: Unless required by applicable law or agreed to in writing, software
+:: distributed under the License is distributed on an "AS IS" BASIS,
+:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+:: See the License for the specific language governing permissions and
+:: limitations under the License.
+
+:: JAVA classpath
+:: Use the local variable CLASSPATH to add custom entries (e.g. JDBC drivers) to
+:: the classpath. Separate multiple paths with ":". Enclose the value
+:: in double quotes. Adding additional files or locations on separate
+:: lines makes things clearer.
+:: Note: If under running under cygwin use "/cygdrive/c/..." for "C:/..."
+:: Example:
+::
+:: Set the CLASSPATH to a jar file and a directory. Note that
+:: "classes dir" is a directory of class files with a space in the name.
+::
+:: CLASSPATH="usr/local/Product1/lib/product.jar"
+:: CLASSPATH="${CLASSPATH}:../MyProject/classes dir"
+::
+SET CLASSPATH="@classpath@"
+
+:: JVM parameters
+:: If you want to modify the default parameters (e.g. maximum heap size -Xmx)
+:: for the Java virtual machine set the local variable JVM_PARAMETERS below
+:: Example:
+:: JVM_PARAMETERS=-Xms100M -Xmx200M
+::
+:: Below are the JVM parameters needed to do remote debugging using Intellij
+:: IDEA. Uncomment and then do: JVM_PARAMETERS="$IDEA_REMOTE_DEBUG_PARAMS"
+:: IDEA_REMOTE_DEBUG_PARAMS="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"
+::
+:: JVM_PARAMETERS=
+
+:: ---------------------------------------------------------------------------
+:: Default configuration. Do not modify below this line.
+:: ---------------------------------------------------------------------------
+:: Application specific parameters
+
+SET MAIN_CLASS=@main.class@
+SET JVM_PARAMS=@jvm.params@
+
+:: Try to find java virtual machine
+IF NOT DEFINED JAVA (
+ IF NOT DEFINED JAVA_HOME SET JAVA="java.exe"
+ IF DEFINED JAVA_HOME SET JAVA="%JAVA_HOME%\bin\java.exe"
+)
+
+:: Run program
+%JAVA% %JVM_PARAMS% %JVM_PARAMETERS% -classpath %CLASSPATH% %MAIN_CLASS% %*
diff --git a/hyracks-core/src/main/scripts/run.sh b/hyracks-core/src/main/scripts/run.sh
new file mode 100644
index 0000000..df04b73
--- /dev/null
+++ b/hyracks-core/src/main/scripts/run.sh
@@ -0,0 +1,100 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# JAVA classpath
+# Use the local variable CLASSPATH to add custom entries (e.g. JDBC drivers) to
+# the classpath. Separate multiple paths with ":". Enclose the value
+# in double quotes. Adding additional files or locations on separate
+# lines makes things clearer.
+# Note: If under running under cygwin use "/cygdrive/c/..." for "C:/..."
+# Example:
+#
+# Set the CLASSPATH to a jar file and a directory. Note that
+# "classes dir" is a directory of class files with a space in the name.
+#
+# CLASSPATH="usr/local/Product1/lib/product.jar"
+# CLASSPATH="${CLASSPATH}:../MyProject/classes dir"
+#
+if [ -z "$CLASSPATH" ]; then
+ CLASSPATH="@classpath@"
+else
+ CLASSPATH="@classpath@:$CLASSPATH"
+fi
+
+# JVM parameters
+# If you want to modify the default parameters (e.g. maximum heap size -Xmx)
+# for the Java virtual machine set the local variable JVM_PARAMETERS below
+# Example:
+# JVM_PARAMETERS=-Xms100M -Xmx200M
+#
+# Below are the JVM parameters needed to do remote debugging using Intellij
+# IDEA. Uncomment and then do: JVM_PARAMETERS="$IDEA_REMOTE_DEBUG_PARAMS"
+# IDEA_REMOTE_DEBUG_PARAMS="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"
+#
+# JVM_PARAMETERS=
+
+#run with shared memory setup
+#if [ -n "${RUN_SHARED_MEM}"]; then
+# JVM_PARAMETERS="${JVM_PARAMETERS} -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_shmem,server=n,address=javadebug,suspend=y"
+#fi
+
+# ---------------------------------------------------------------------------
+# Default configuration. Do not modify below this line.
+# ---------------------------------------------------------------------------
+# Application specific parameters
+
+MAIN_CLASS="@main.class@"
+JVM_PARAMS="@jvm.params@"
+
+# Cygwin support. $cygwin _must_ be set to either true or false.
+case "`uname`" in
+ CYGWIN*) cygwin=true ;;
+ *) cygwin=false ;;
+esac
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched
+if $cygwin; then
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
+ [ -n "$CLASSPATH" ] &&
+ CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
+fi
+
+# Try to find java virtual machine
+if [ -z "${JAVA}" ]; then
+ if [ -z "${JAVA_HOME}" ]; then
+ JAVA=java
+ else
+ JAVA=${JAVA_HOME}/bin/java
+ fi
+fi
+
+# Try to find directory where this script is located
+COMMAND="${PWD}/$0"
+if [ ! -f "${COMMAND}" ]; then
+ COMMAND="$0"
+fi
+BASEDIR=`expr "${COMMAND}" : '\(.*\)/\.*'`
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin; then
+# JAVA=`cygpath --path --windows "$JAVA"`
+ CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
+fi
+
+# Run program
+${JAVA} ${JVM_PARAMS} ${JVM_PARAMETERS} -classpath "${CLASSPATH}" ${MAIN_CLASS} $*
diff --git a/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/comm/SerializationDeserializationTest.java b/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/comm/SerializationDeserializationTest.java
new file mode 100644
index 0000000..1467de5
--- /dev/null
+++ b/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/comm/SerializationDeserializationTest.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.tests.comm;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.IDataWriter;
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataReader;
+import edu.uci.ics.hyracks.api.dataflow.IOpenableDataWriter;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.comm.io.FrameDeserializingDataReader;
+import edu.uci.ics.hyracks.comm.io.SerializingDataWriter;
+import edu.uci.ics.hyracks.context.HyracksContext;
+import edu.uci.ics.hyracks.coreops.data.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.coreops.data.StringSerializerDeserializer;
+
+public class SerializationDeserializationTest {
+ private static final String DBLP_FILE = "data/dblp.txt";
+
+ private static class SerDeserRunner {
+ private final HyracksContext ctx;
+ private static final int FRAME_SIZE = 32768;
+ private RecordDescriptor rDes;
+ private List<ByteBuffer> buffers;
+
+ public SerDeserRunner(RecordDescriptor rDes) {
+ ctx = new HyracksContext(FRAME_SIZE);
+ this.rDes = rDes;
+ buffers = new ArrayList<ByteBuffer>();
+ }
+
+ public IOpenableDataWriter<Object[]> createWriter() {
+ return new SerializingDataWriter(ctx, rDes, new IFrameWriter() {
+ @Override
+ public void open() throws HyracksDataException {
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ ByteBuffer toBuf = ctx.getResourceManager().allocateFrame();
+ toBuf.put(buffer);
+ buffers.add(toBuf);
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+
+ }
+ });
+ }
+
+ public IOpenableDataReader<Object[]> createDataReader() {
+ return new FrameDeserializingDataReader(ctx, new IFrameReader() {
+ private int i;
+
+ @Override
+ public void open() throws HyracksDataException {
+ i = 0;
+ }
+
+ @Override
+ public boolean nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ if (i < buffers.size()) {
+ ByteBuffer buf = buffers.get(i);
+ buf.flip();
+ buffer.put(buf);
+ buffer.flip();
+ ++i;
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+
+ }
+ }, rDes);
+ }
+ }
+
+ private interface LineProcessor {
+ public void process(String line, IDataWriter<Object[]> writer) throws Exception;
+ }
+
+ private void run(RecordDescriptor rDes, LineProcessor lp) throws Exception {
+ SerDeserRunner runner = new SerDeserRunner(rDes);
+ IOpenableDataWriter<Object[]> writer = runner.createWriter();
+ writer.open();
+ BufferedReader in = new BufferedReader(new FileReader(DBLP_FILE));
+ String line;
+ while ((line = in.readLine()) != null) {
+ lp.process(line, writer);
+ }
+ writer.close();
+
+ IOpenableDataReader<Object[]> reader = runner.createDataReader();
+ reader.open();
+ Object[] arr;
+ while ((arr = reader.readData()) != null) {
+ System.err.println(arr[0] + " " + arr[1]);
+ }
+ reader.close();
+ }
+
+ @Test
+ public void serdeser01() throws Exception {
+ RecordDescriptor rDes = new RecordDescriptor(new ISerializerDeserializer[] {
+ StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+ LineProcessor processor = new LineProcessor() {
+ @Override
+ public void process(String line, IDataWriter<Object[]> writer) throws Exception {
+ String[] splits = line.split(" ");
+ for (String s : splits) {
+ writer.writeData(new Object[] { s, Integer.valueOf(1) });
+ }
+ }
+ };
+ run(rDes, processor);
+ }
+}
diff --git a/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java b/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
new file mode 100644
index 0000000..94bc57a
--- /dev/null
+++ b/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.tests.integration;
+
+import java.util.EnumSet;
+import java.util.UUID;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.config.CCConfig;
+import edu.uci.ics.hyracks.config.NCConfig;
+import edu.uci.ics.hyracks.controller.ClusterControllerService;
+import edu.uci.ics.hyracks.controller.NodeControllerService;
+
+public abstract class AbstractIntegrationTest {
+ public static final String NC1_ID = "nc1";
+ public static final String NC2_ID = "nc2";
+
+ protected static ClusterControllerService cc;
+ protected static NodeControllerService nc1;
+ protected static NodeControllerService nc2;
+
+ @BeforeClass
+ public static void init() throws Exception {
+ CCConfig ccConfig = new CCConfig();
+ ccConfig.port = 39001;
+ cc = new ClusterControllerService(ccConfig);
+ cc.start();
+
+ NCConfig ncConfig1 = new NCConfig();
+ ncConfig1.ccHost = "localhost";
+ ncConfig1.ccPort = 39001;
+ ncConfig1.dataIPAddress = "127.0.0.1";
+ ncConfig1.nodeId = NC1_ID;
+ nc1 = new NodeControllerService(ncConfig1);
+ nc1.start();
+
+ NCConfig ncConfig2 = new NCConfig();
+ ncConfig2.ccHost = "localhost";
+ ncConfig2.ccPort = 39001;
+ ncConfig2.dataIPAddress = "127.0.0.1";
+ ncConfig2.nodeId = NC2_ID;
+ nc2 = new NodeControllerService(ncConfig2);
+ nc2.start();
+ }
+
+ @AfterClass
+ public static void deinit() throws Exception {
+ nc2.stop();
+ nc1.stop();
+ cc.stop();
+ }
+
+ void runTest(JobSpecification spec) throws Exception {
+ UUID jobId = cc.createJob(spec, EnumSet.of(JobFlag.COLLECT_FRAME_COUNTS));
+ System.err.println(spec.toJSON());
+ cc.start(jobId);
+ System.err.print(jobId);
+ System.err.println(cc.waitForCompletion(jobId));
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/integration/CountOfCountsTest.java b/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/integration/CountOfCountsTest.java
new file mode 100644
index 0000000..a3beb11
--- /dev/null
+++ b/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/integration/CountOfCountsTest.java
@@ -0,0 +1,252 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.tests.integration;
+
+import java.io.File;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.constraints.AbsoluteLocationConstraint;
+import edu.uci.ics.hyracks.api.constraints.LocationConstraint;
+import edu.uci.ics.hyracks.api.constraints.PartitionConstraint;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.coreops.ExternalSortOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.coreops.InMemorySortOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.MToNHashPartitioningConnectorDescriptor;
+import edu.uci.ics.hyracks.coreops.MToNReplicatingConnectorDescriptor;
+import edu.uci.ics.hyracks.coreops.OneToOneConnectorDescriptor;
+import edu.uci.ics.hyracks.coreops.PrinterOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.aggregators.SumStringGroupAggregator;
+import edu.uci.ics.hyracks.coreops.data.StringBinaryComparatorFactory;
+import edu.uci.ics.hyracks.coreops.data.StringBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.coreops.data.StringComparatorFactory;
+import edu.uci.ics.hyracks.coreops.data.StringSerializerDeserializer;
+import edu.uci.ics.hyracks.coreops.file.CSVFileScanOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.file.FileSplit;
+import edu.uci.ics.hyracks.coreops.group.PreclusteredGroupOperatorDescriptor;
+
+public class CountOfCountsTest extends AbstractIntegrationTest {
+ @Test
+ public void countOfCountsSingleNC() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileSplit[] splits = new FileSplit[] { new FileSplit(NC1_ID, new File("data/words.txt")) };
+ RecordDescriptor desc = new RecordDescriptor(
+ new ISerializerDeserializer[] { StringSerializerDeserializer.INSTANCE });
+
+ CSVFileScanOperatorDescriptor csvScanner = new CSVFileScanOperatorDescriptor(spec, splits, desc);
+ PartitionConstraint csvPartitionConstraint = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ csvScanner.setPartitionConstraint(csvPartitionConstraint);
+
+ InMemorySortOperatorDescriptor sorter = new InMemorySortOperatorDescriptor(spec, new int[] { 0 },
+ new IBinaryComparatorFactory[] { StringBinaryComparatorFactory.INSTANCE }, desc);
+ PartitionConstraint sorterPartitionConstraint = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ sorter.setPartitionConstraint(sorterPartitionConstraint);
+
+ RecordDescriptor desc2 = new RecordDescriptor(new ISerializerDeserializer[] {
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE });
+ PreclusteredGroupOperatorDescriptor group = new PreclusteredGroupOperatorDescriptor(spec, new int[] { 0 },
+ new IComparatorFactory[] { StringComparatorFactory.INSTANCE }, new SumStringGroupAggregator(0), desc2);
+ PartitionConstraint groupPartitionConstraint = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ group.setPartitionConstraint(groupPartitionConstraint);
+
+ InMemorySortOperatorDescriptor sorter2 = new InMemorySortOperatorDescriptor(spec, new int[] { 1 },
+ new IBinaryComparatorFactory[] { StringBinaryComparatorFactory.INSTANCE }, desc2);
+ PartitionConstraint sorterPartitionConstraint2 = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ sorter2.setPartitionConstraint(sorterPartitionConstraint2);
+
+ PreclusteredGroupOperatorDescriptor group2 = new PreclusteredGroupOperatorDescriptor(spec, new int[] { 1 },
+ new IComparatorFactory[] { StringComparatorFactory.INSTANCE }, new SumStringGroupAggregator(1), desc2);
+ PartitionConstraint groupPartitionConstraint2 = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ group2.setPartitionConstraint(groupPartitionConstraint2);
+
+ PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ PartitionConstraint printerPartitionConstraint = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ printer.setPartitionConstraint(printerPartitionConstraint);
+
+ IConnectorDescriptor conn1 = new MToNHashPartitioningConnectorDescriptor(spec,
+ new FieldHashPartitionComputerFactory(new int[] { 0 },
+ new IBinaryHashFunctionFactory[] { StringBinaryHashFunctionFactory.INSTANCE }));
+ spec.connect(conn1, csvScanner, 0, sorter, 0);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, sorter, 0, group, 0);
+
+ IConnectorDescriptor conn3 = new MToNHashPartitioningConnectorDescriptor(spec,
+ new FieldHashPartitionComputerFactory(new int[] { 1 },
+ new IBinaryHashFunctionFactory[] { StringBinaryHashFunctionFactory.INSTANCE }));
+ spec.connect(conn3, group, 0, sorter2, 0);
+
+ IConnectorDescriptor conn4 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn4, sorter2, 0, group2, 0);
+
+ IConnectorDescriptor conn5 = new MToNReplicatingConnectorDescriptor(spec);
+ spec.connect(conn5, group2, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void countOfCountsMultiNC() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileSplit[] splits = new FileSplit[] { new FileSplit(NC1_ID, new File("data/words.txt")) };
+ RecordDescriptor desc = new RecordDescriptor(
+ new ISerializerDeserializer[] { StringSerializerDeserializer.INSTANCE });
+
+ CSVFileScanOperatorDescriptor csvScanner = new CSVFileScanOperatorDescriptor(spec, splits, desc);
+ PartitionConstraint csvPartitionConstraint = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ csvScanner.setPartitionConstraint(csvPartitionConstraint);
+
+ InMemorySortOperatorDescriptor sorter = new InMemorySortOperatorDescriptor(spec, new int[] { 0 },
+ new IBinaryComparatorFactory[] { StringBinaryComparatorFactory.INSTANCE }, desc);
+ PartitionConstraint sorterPartitionConstraint = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID),
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) });
+ sorter.setPartitionConstraint(sorterPartitionConstraint);
+
+ RecordDescriptor desc2 = new RecordDescriptor(new ISerializerDeserializer[] {
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE });
+ PreclusteredGroupOperatorDescriptor group = new PreclusteredGroupOperatorDescriptor(spec, new int[] { 0 },
+ new IComparatorFactory[] { StringComparatorFactory.INSTANCE }, new SumStringGroupAggregator(0), desc2);
+ PartitionConstraint groupPartitionConstraint = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID),
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) });
+ group.setPartitionConstraint(groupPartitionConstraint);
+
+ InMemorySortOperatorDescriptor sorter2 = new InMemorySortOperatorDescriptor(spec, new int[] { 1 },
+ new IBinaryComparatorFactory[] { StringBinaryComparatorFactory.INSTANCE }, desc2);
+ PartitionConstraint sorterPartitionConstraint2 = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) });
+ sorter2.setPartitionConstraint(sorterPartitionConstraint2);
+
+ PreclusteredGroupOperatorDescriptor group2 = new PreclusteredGroupOperatorDescriptor(spec, new int[] { 1 },
+ new IComparatorFactory[] { StringComparatorFactory.INSTANCE }, new SumStringGroupAggregator(1), desc2);
+ PartitionConstraint groupPartitionConstraint2 = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) });
+ group2.setPartitionConstraint(groupPartitionConstraint2);
+
+ PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ PartitionConstraint printerPartitionConstraint = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ printer.setPartitionConstraint(printerPartitionConstraint);
+
+ IConnectorDescriptor conn1 = new MToNHashPartitioningConnectorDescriptor(spec,
+ new FieldHashPartitionComputerFactory(new int[] { 0 },
+ new IBinaryHashFunctionFactory[] { StringBinaryHashFunctionFactory.INSTANCE }));
+ spec.connect(conn1, csvScanner, 0, sorter, 0);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, sorter, 0, group, 0);
+
+ IConnectorDescriptor conn3 = new MToNHashPartitioningConnectorDescriptor(spec,
+ new FieldHashPartitionComputerFactory(new int[] { 1 },
+ new IBinaryHashFunctionFactory[] { StringBinaryHashFunctionFactory.INSTANCE }));
+ spec.connect(conn3, group, 0, sorter2, 0);
+
+ IConnectorDescriptor conn4 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn4, sorter2, 0, group2, 0);
+
+ IConnectorDescriptor conn5 = new MToNReplicatingConnectorDescriptor(spec);
+ spec.connect(conn5, group2, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void countOfCountsExternalSortMultiNC() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileSplit[] splits = new FileSplit[] { new FileSplit(NC1_ID, new File("data/words.txt")) };
+ RecordDescriptor desc = new RecordDescriptor(
+ new ISerializerDeserializer[] { StringSerializerDeserializer.INSTANCE });
+
+ CSVFileScanOperatorDescriptor csvScanner = new CSVFileScanOperatorDescriptor(spec, splits, desc);
+ PartitionConstraint csvPartitionConstraint = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ csvScanner.setPartitionConstraint(csvPartitionConstraint);
+
+ ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, 3, new int[] { 0 },
+ new IBinaryComparatorFactory[] { StringBinaryComparatorFactory.INSTANCE }, desc);
+ PartitionConstraint sorterPartitionConstraint = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID),
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) });
+ sorter.setPartitionConstraint(sorterPartitionConstraint);
+
+ RecordDescriptor desc2 = new RecordDescriptor(new ISerializerDeserializer[] {
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE });
+ PreclusteredGroupOperatorDescriptor group = new PreclusteredGroupOperatorDescriptor(spec, new int[] { 0 },
+ new IComparatorFactory[] { StringComparatorFactory.INSTANCE }, new SumStringGroupAggregator(0), desc2);
+ PartitionConstraint groupPartitionConstraint = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID),
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) });
+ group.setPartitionConstraint(groupPartitionConstraint);
+
+ ExternalSortOperatorDescriptor sorter2 = new ExternalSortOperatorDescriptor(spec, 3, new int[] { 1 },
+ new IBinaryComparatorFactory[] { StringBinaryComparatorFactory.INSTANCE }, desc2);
+ PartitionConstraint sorterPartitionConstraint2 = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) });
+ sorter2.setPartitionConstraint(sorterPartitionConstraint2);
+
+ PreclusteredGroupOperatorDescriptor group2 = new PreclusteredGroupOperatorDescriptor(spec, new int[] { 1 },
+ new IComparatorFactory[] { StringComparatorFactory.INSTANCE }, new SumStringGroupAggregator(1), desc2);
+ PartitionConstraint groupPartitionConstraint2 = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) });
+ group2.setPartitionConstraint(groupPartitionConstraint2);
+
+ PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ PartitionConstraint printerPartitionConstraint = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ printer.setPartitionConstraint(printerPartitionConstraint);
+
+ IConnectorDescriptor conn1 = new MToNHashPartitioningConnectorDescriptor(spec,
+ new FieldHashPartitionComputerFactory(new int[] { 0 },
+ new IBinaryHashFunctionFactory[] { StringBinaryHashFunctionFactory.INSTANCE }));
+ spec.connect(conn1, csvScanner, 0, sorter, 0);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, sorter, 0, group, 0);
+
+ IConnectorDescriptor conn3 = new MToNHashPartitioningConnectorDescriptor(spec,
+ new FieldHashPartitionComputerFactory(new int[] { 1 },
+ new IBinaryHashFunctionFactory[] { StringBinaryHashFunctionFactory.INSTANCE }));
+ spec.connect(conn3, group, 0, sorter2, 0);
+
+ IConnectorDescriptor conn4 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn4, sorter2, 0, group2, 0);
+
+ IConnectorDescriptor conn5 = new MToNReplicatingConnectorDescriptor(spec);
+ spec.connect(conn5, group2, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/integration/ScanPrintTest.java b/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/integration/ScanPrintTest.java
new file mode 100644
index 0000000..d619bba
--- /dev/null
+++ b/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/integration/ScanPrintTest.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.tests.integration;
+
+import java.io.File;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.constraints.AbsoluteLocationConstraint;
+import edu.uci.ics.hyracks.api.constraints.LocationConstraint;
+import edu.uci.ics.hyracks.api.constraints.PartitionConstraint;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.coreops.OneToOneConnectorDescriptor;
+import edu.uci.ics.hyracks.coreops.PrinterOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.data.StringSerializerDeserializer;
+import edu.uci.ics.hyracks.coreops.file.CSVFileScanOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.file.FileSplit;
+
+public class ScanPrintTest extends AbstractIntegrationTest {
+ @Test
+ public void scanPrint01() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileSplit[] splits = new FileSplit[] { new FileSplit(NC2_ID, new File("data/words.txt")),
+ new FileSplit(NC1_ID, new File("data/words.txt")) };
+ RecordDescriptor desc = new RecordDescriptor(
+ new ISerializerDeserializer[] { StringSerializerDeserializer.INSTANCE });
+
+ CSVFileScanOperatorDescriptor csvScanner = new CSVFileScanOperatorDescriptor(spec, splits, desc);
+ PartitionConstraint csvPartitionConstraint = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC2_ID), new AbsoluteLocationConstraint(NC1_ID) });
+ csvScanner.setPartitionConstraint(csvPartitionConstraint);
+
+ PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ PartitionConstraint printerPartitionConstraint = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC2_ID), new AbsoluteLocationConstraint(NC1_ID) });
+ printer.setPartitionConstraint(printerPartitionConstraint);
+
+ IConnectorDescriptor conn = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn, csvScanner, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/integration/SortMergeTest.java b/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/integration/SortMergeTest.java
new file mode 100644
index 0000000..cd254e2
--- /dev/null
+++ b/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/integration/SortMergeTest.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.tests.integration;
+
+import java.io.File;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.constraints.AbsoluteLocationConstraint;
+import edu.uci.ics.hyracks.api.constraints.LocationConstraint;
+import edu.uci.ics.hyracks.api.constraints.PartitionConstraint;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.coreops.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.coreops.InMemorySortOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.MToNHashPartitioningMergingConnectorDescriptor;
+import edu.uci.ics.hyracks.coreops.OneToOneConnectorDescriptor;
+import edu.uci.ics.hyracks.coreops.PrinterOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.data.StringBinaryComparatorFactory;
+import edu.uci.ics.hyracks.coreops.data.StringBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.coreops.data.StringSerializerDeserializer;
+import edu.uci.ics.hyracks.coreops.file.CSVFileScanOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.file.FileSplit;
+
+public class SortMergeTest extends AbstractIntegrationTest {
+ @Test
+ public void sortMergeTest01() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileSplit[] ordersSplits = new FileSplit[] {
+ new FileSplit(NC1_ID, new File("data/tpch0.001/orders-part1.tbl")),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/orders-part2.tbl")) };
+ RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE });
+
+ CSVFileScanOperatorDescriptor ordScanner = new CSVFileScanOperatorDescriptor(spec, ordersSplits, ordersDesc,
+ '|', "'\"");
+ PartitionConstraint ordersPartitionConstraint = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) });
+ ordScanner.setPartitionConstraint(ordersPartitionConstraint);
+
+ InMemorySortOperatorDescriptor sorter = new InMemorySortOperatorDescriptor(spec, new int[] { 1 },
+ new IBinaryComparatorFactory[] { StringBinaryComparatorFactory.INSTANCE }, ordersDesc);
+ PartitionConstraint sortersPartitionConstraint = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) });
+ sorter.setPartitionConstraint(sortersPartitionConstraint);
+
+ PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ PartitionConstraint printerPartitionConstraint = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ printer.setPartitionConstraint(printerPartitionConstraint);
+
+ spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
+
+ spec.connect(new MToNHashPartitioningMergingConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(
+ new int[] { 1 }, new IBinaryHashFunctionFactory[] { StringBinaryHashFunctionFactory.INSTANCE }),
+ new int[] { 1 }, new IBinaryComparatorFactory[] { StringBinaryComparatorFactory.INSTANCE }), sorter, 0,
+ printer, 0);
+
+ runTest(spec);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java b/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java
new file mode 100644
index 0000000..593eac1
--- /dev/null
+++ b/hyracks-core/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java
@@ -0,0 +1,300 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.tests.integration;
+
+import java.io.File;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.constraints.AbsoluteLocationConstraint;
+import edu.uci.ics.hyracks.api.constraints.LocationConstraint;
+import edu.uci.ics.hyracks.api.constraints.PartitionConstraint;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.coreops.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.coreops.MToNHashPartitioningConnectorDescriptor;
+import edu.uci.ics.hyracks.coreops.MToNReplicatingConnectorDescriptor;
+import edu.uci.ics.hyracks.coreops.MaterializingOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.OneToOneConnectorDescriptor;
+import edu.uci.ics.hyracks.coreops.PrinterOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.data.StringBinaryComparatorFactory;
+import edu.uci.ics.hyracks.coreops.data.StringBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.coreops.data.StringSerializerDeserializer;
+import edu.uci.ics.hyracks.coreops.file.CSVFileScanOperatorDescriptor;
+import edu.uci.ics.hyracks.coreops.file.FileSplit;
+import edu.uci.ics.hyracks.coreops.join.InMemoryHashJoinOperatorDescriptor;
+
+public class TPCHCustomerOrderHashJoinTest extends AbstractIntegrationTest {
+ /*
+ * TPCH Customer table:
+ * CREATE TABLE CUSTOMER (
+ * C_CUSTKEY INTEGER NOT NULL,
+ * C_NAME VARCHAR(25) NOT NULL,
+ * C_ADDRESS VARCHAR(40) NOT NULL,
+ * C_NATIONKEY INTEGER NOT NULL,
+ * C_PHONE CHAR(15) NOT NULL,
+ * C_ACCTBAL DECIMAL(15,2) NOT NULL,
+ * C_MKTSEGMENT CHAR(10) NOT NULL,
+ * C_COMMENT VARCHAR(117) NOT NULL
+ * );
+ * TPCH Orders table:
+ * CREATE TABLE ORDERS (
+ * O_ORDERKEY INTEGER NOT NULL,
+ * O_CUSTKEY INTEGER NOT NULL,
+ * O_ORDERSTATUS CHAR(1) NOT NULL,
+ * O_TOTALPRICE DECIMAL(15,2) NOT NULL,
+ * O_ORDERDATE DATE NOT NULL,
+ * O_ORDERPRIORITY CHAR(15) NOT NULL,
+ * O_CLERK CHAR(15) NOT NULL,
+ * O_SHIPPRIORITY INTEGER NOT NULL,
+ * O_COMMENT VARCHAR(79) NOT NULL
+ * );
+ */
+
+ @Test
+ public void customerOrderCIDJoin() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new File("data/tpch0.001/customer.tbl")) };
+ RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE });
+
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new File("data/tpch0.001/orders.tbl")) };
+ RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE });
+
+ RecordDescriptor custOrderJoinDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE });
+
+ CSVFileScanOperatorDescriptor ordScanner = new CSVFileScanOperatorDescriptor(spec, ordersSplits, ordersDesc,
+ '|', "'\"");
+ PartitionConstraint ordersPartitionConstraint = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ ordScanner.setPartitionConstraint(ordersPartitionConstraint);
+
+ CSVFileScanOperatorDescriptor custScanner = new CSVFileScanOperatorDescriptor(spec, custSplits, custDesc, '|',
+ "'\"");
+ PartitionConstraint custPartitionConstraint = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ custScanner.setPartitionConstraint(custPartitionConstraint);
+
+ InMemoryHashJoinOperatorDescriptor join = new InMemoryHashJoinOperatorDescriptor(spec, new int[] { 1 },
+ new int[] { 0 }, new IBinaryHashFunctionFactory[] { StringBinaryHashFunctionFactory.INSTANCE },
+ new IBinaryComparatorFactory[] { StringBinaryComparatorFactory.INSTANCE }, custOrderJoinDesc, 128);
+ PartitionConstraint joinPartitionConstraint = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ join.setPartitionConstraint(joinPartitionConstraint);
+
+ PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ PartitionConstraint printerPartitionConstraint = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ printer.setPartitionConstraint(printerPartitionConstraint);
+
+ IConnectorDescriptor ordJoinConn = new OneToOneConnectorDescriptor(spec);
+ spec.connect(ordJoinConn, ordScanner, 0, join, 0);
+
+ IConnectorDescriptor custJoinConn = new OneToOneConnectorDescriptor(spec);
+ spec.connect(custJoinConn, custScanner, 0, join, 1);
+
+ IConnectorDescriptor joinPrinterConn = new OneToOneConnectorDescriptor(spec);
+ spec.connect(joinPrinterConn, join, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void customerOrderCIDJoinMulti() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileSplit[] custSplits = new FileSplit[] {
+ new FileSplit(NC1_ID, new File("data/tpch0.001/customer-part1.tbl")),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/customer-part2.tbl")) };
+ RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE });
+
+ FileSplit[] ordersSplits = new FileSplit[] {
+ new FileSplit(NC1_ID, new File("data/tpch0.001/orders-part1.tbl")),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/orders-part2.tbl")) };
+ RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE });
+
+ RecordDescriptor custOrderJoinDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE });
+
+ CSVFileScanOperatorDescriptor ordScanner = new CSVFileScanOperatorDescriptor(spec, ordersSplits, ordersDesc,
+ '|', "'\"");
+ PartitionConstraint ordersPartitionConstraint = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) });
+ ordScanner.setPartitionConstraint(ordersPartitionConstraint);
+
+ CSVFileScanOperatorDescriptor custScanner = new CSVFileScanOperatorDescriptor(spec, custSplits, custDesc, '|',
+ "'\"");
+ PartitionConstraint custPartitionConstraint = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) });
+ custScanner.setPartitionConstraint(custPartitionConstraint);
+
+ InMemoryHashJoinOperatorDescriptor join = new InMemoryHashJoinOperatorDescriptor(spec, new int[] { 1 },
+ new int[] { 0 }, new IBinaryHashFunctionFactory[] { StringBinaryHashFunctionFactory.INSTANCE },
+ new IBinaryComparatorFactory[] { StringBinaryComparatorFactory.INSTANCE }, custOrderJoinDesc, 128);
+ PartitionConstraint joinPartitionConstraint = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) });
+ join.setPartitionConstraint(joinPartitionConstraint);
+
+ PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ PartitionConstraint printerPartitionConstraint = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ printer.setPartitionConstraint(printerPartitionConstraint);
+
+ IConnectorDescriptor ordJoinConn = new MToNHashPartitioningConnectorDescriptor(spec,
+ new FieldHashPartitionComputerFactory(new int[] { 1 },
+ new IBinaryHashFunctionFactory[] { StringBinaryHashFunctionFactory.INSTANCE }));
+ spec.connect(ordJoinConn, ordScanner, 0, join, 0);
+
+ IConnectorDescriptor custJoinConn = new MToNHashPartitioningConnectorDescriptor(spec,
+ new FieldHashPartitionComputerFactory(new int[] { 0 },
+ new IBinaryHashFunctionFactory[] { StringBinaryHashFunctionFactory.INSTANCE }));
+ spec.connect(custJoinConn, custScanner, 0, join, 1);
+
+ IConnectorDescriptor joinPrinterConn = new MToNReplicatingConnectorDescriptor(spec);
+ spec.connect(joinPrinterConn, join, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void customerOrderCIDJoinMultiMaterialized() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileSplit[] custSplits = new FileSplit[] {
+ new FileSplit(NC1_ID, new File("data/tpch0.001/customer-part1.tbl")),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/customer-part2.tbl")) };
+ RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE });
+
+ FileSplit[] ordersSplits = new FileSplit[] {
+ new FileSplit(NC1_ID, new File("data/tpch0.001/orders-part1.tbl")),
+ new FileSplit(NC2_ID, new File("data/tpch0.001/orders-part2.tbl")) };
+ RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE });
+
+ RecordDescriptor custOrderJoinDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE, StringSerializerDeserializer.INSTANCE,
+ StringSerializerDeserializer.INSTANCE });
+
+ CSVFileScanOperatorDescriptor ordScanner = new CSVFileScanOperatorDescriptor(spec, ordersSplits, ordersDesc,
+ '|', "'\"");
+ PartitionConstraint ordersPartitionConstraint = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) });
+ ordScanner.setPartitionConstraint(ordersPartitionConstraint);
+
+ CSVFileScanOperatorDescriptor custScanner = new CSVFileScanOperatorDescriptor(spec, custSplits, custDesc, '|',
+ "'\"");
+ PartitionConstraint custPartitionConstraint = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) });
+ custScanner.setPartitionConstraint(custPartitionConstraint);
+
+ MaterializingOperatorDescriptor ordMat = new MaterializingOperatorDescriptor(spec, ordersDesc);
+ ordMat.setPartitionConstraint(new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) }));
+
+ MaterializingOperatorDescriptor custMat = new MaterializingOperatorDescriptor(spec, custDesc);
+ custMat.setPartitionConstraint(new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) }));
+
+ InMemoryHashJoinOperatorDescriptor join = new InMemoryHashJoinOperatorDescriptor(spec, new int[] { 1 },
+ new int[] { 0 }, new IBinaryHashFunctionFactory[] { StringBinaryHashFunctionFactory.INSTANCE },
+ new IBinaryComparatorFactory[] { StringBinaryComparatorFactory.INSTANCE }, custOrderJoinDesc, 128);
+ PartitionConstraint joinPartitionConstraint = new PartitionConstraint(new LocationConstraint[] {
+ new AbsoluteLocationConstraint(NC1_ID), new AbsoluteLocationConstraint(NC2_ID) });
+ join.setPartitionConstraint(joinPartitionConstraint);
+
+ PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ PartitionConstraint printerPartitionConstraint = new PartitionConstraint(
+ new LocationConstraint[] { new AbsoluteLocationConstraint(NC1_ID) });
+ printer.setPartitionConstraint(printerPartitionConstraint);
+
+ IConnectorDescriptor ordPartConn = new MToNHashPartitioningConnectorDescriptor(spec,
+ new FieldHashPartitionComputerFactory(new int[] { 1 },
+ new IBinaryHashFunctionFactory[] { StringBinaryHashFunctionFactory.INSTANCE }));
+ spec.connect(ordPartConn, ordScanner, 0, ordMat, 0);
+
+ IConnectorDescriptor custPartConn = new MToNHashPartitioningConnectorDescriptor(spec,
+ new FieldHashPartitionComputerFactory(new int[] { 0 },
+ new IBinaryHashFunctionFactory[] { StringBinaryHashFunctionFactory.INSTANCE }));
+ spec.connect(custPartConn, custScanner, 0, custMat, 0);
+
+ IConnectorDescriptor ordJoinConn = new OneToOneConnectorDescriptor(spec);
+ spec.connect(ordJoinConn, ordMat, 0, join, 0);
+
+ IConnectorDescriptor custJoinConn = new OneToOneConnectorDescriptor(spec);
+ spec.connect(custJoinConn, custMat, 0, join, 1);
+
+ IConnectorDescriptor joinPrinterConn = new MToNReplicatingConnectorDescriptor(spec);
+ spec.connect(joinPrinterConn, join, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+}
\ No newline at end of file