Optimized in-memory sorting. Added Eclipse project files
git-svn-id: https://hyracks.googlecode.com/svn/trunk/hyracks@148 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-examples/btree-example/.project b/hyracks-examples/btree-example/.project
new file mode 100644
index 0000000..ef68a5a
--- /dev/null
+++ b/hyracks-examples/btree-example/.project
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>btree-example</name>
+ <comment></comment>
+ <projects>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.maven.ide.eclipse.maven2Builder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.maven.ide.eclipse.maven2Nature</nature>
+ </natures>
+</projectDescription>
diff --git a/hyracks-examples/btree-example/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/btree-example/.settings/org.maven.ide.eclipse.prefs
new file mode 100644
index 0000000..6733f8d
--- /dev/null
+++ b/hyracks-examples/btree-example/.settings/org.maven.ide.eclipse.prefs
@@ -0,0 +1,9 @@
+#Wed Oct 06 08:06:48 PDT 2010
+activeProfiles=
+eclipse.preferences.version=1
+fullBuildGoals=process-test-resources
+includeModules=false
+resolveWorkspaceProjects=true
+resourceFilterGoals=process-resources resources\:testResources
+skipCompilerPlugin=true
+version=1
diff --git a/hyracks-examples/btree-example/btreeapp/.classpath b/hyracks-examples/btree-example/btreeapp/.classpath
new file mode 100644
index 0000000..1a0f9bf
--- /dev/null
+++ b/hyracks-examples/btree-example/btreeapp/.classpath
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.4"/>
+ <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
+ <classpathentry kind="output" path="target/android-classes"/>
+</classpath>
diff --git a/hyracks-examples/btree-example/btreeapp/.project b/hyracks-examples/btree-example/btreeapp/.project
new file mode 100644
index 0000000..897fb85
--- /dev/null
+++ b/hyracks-examples/btree-example/btreeapp/.project
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>btreeapp</name>
+ <comment></comment>
+ <projects>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ <buildCommand>
+ <name>org.maven.ide.eclipse.maven2Builder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ <nature>org.maven.ide.eclipse.maven2Nature</nature>
+ </natures>
+</projectDescription>
diff --git a/hyracks-examples/btree-example/btreeapp/.settings/org.eclipse.jdt.core.prefs b/hyracks-examples/btree-example/btreeapp/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 0000000..fdf1b32
--- /dev/null
+++ b/hyracks-examples/btree-example/btreeapp/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,6 @@
+#Wed Oct 06 08:06:49 PDT 2010
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.4
+org.eclipse.jdt.core.compiler.compliance=1.4
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.source=1.4
diff --git a/hyracks-examples/btree-example/btreeapp/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/btree-example/btreeapp/.settings/org.maven.ide.eclipse.prefs
new file mode 100644
index 0000000..6733f8d
--- /dev/null
+++ b/hyracks-examples/btree-example/btreeapp/.settings/org.maven.ide.eclipse.prefs
@@ -0,0 +1,9 @@
+#Wed Oct 06 08:06:48 PDT 2010
+activeProfiles=
+eclipse.preferences.version=1
+fullBuildGoals=process-test-resources
+includeModules=false
+resolveWorkspaceProjects=true
+resourceFilterGoals=process-resources resources\:testResources
+skipCompilerPlugin=true
+version=1
diff --git a/hyracks-examples/btree-example/btreeclient/.classpath b/hyracks-examples/btree-example/btreeclient/.classpath
new file mode 100644
index 0000000..9b2abd7
--- /dev/null
+++ b/hyracks-examples/btree-example/btreeclient/.classpath
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" output="target/classes" path="src/main/java"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
+ <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
+ <classpathentry kind="output" path="target/android-classes"/>
+</classpath>
diff --git a/hyracks-examples/btree-example/btreeclient/.project b/hyracks-examples/btree-example/btreeclient/.project
new file mode 100644
index 0000000..8e16966
--- /dev/null
+++ b/hyracks-examples/btree-example/btreeclient/.project
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>btreeclient</name>
+ <comment></comment>
+ <projects>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ <buildCommand>
+ <name>org.maven.ide.eclipse.maven2Builder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ <nature>org.maven.ide.eclipse.maven2Nature</nature>
+ </natures>
+</projectDescription>
diff --git a/hyracks-examples/btree-example/btreeclient/.settings/org.eclipse.jdt.core.prefs b/hyracks-examples/btree-example/btreeclient/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 0000000..367ddc4
--- /dev/null
+++ b/hyracks-examples/btree-example/btreeclient/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,6 @@
+#Wed Oct 06 08:06:49 PDT 2010
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
+org.eclipse.jdt.core.compiler.compliance=1.6
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-examples/btree-example/btreeclient/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/btree-example/btreeclient/.settings/org.maven.ide.eclipse.prefs
new file mode 100644
index 0000000..6733f8d
--- /dev/null
+++ b/hyracks-examples/btree-example/btreeclient/.settings/org.maven.ide.eclipse.prefs
@@ -0,0 +1,9 @@
+#Wed Oct 06 08:06:48 PDT 2010
+activeProfiles=
+eclipse.preferences.version=1
+fullBuildGoals=process-test-resources
+includeModules=false
+resolveWorkspaceProjects=true
+resourceFilterGoals=process-resources resources\:testResources
+skipCompilerPlugin=true
+version=1
diff --git a/hyracks-examples/btree-example/btreehelper/.classpath b/hyracks-examples/btree-example/btreehelper/.classpath
new file mode 100644
index 0000000..305dfef
--- /dev/null
+++ b/hyracks-examples/btree-example/btreehelper/.classpath
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" output="target/classes" path="src/main/java"/>
+ <classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
+ <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
+ <classpathentry kind="output" path="target/android-classes"/>
+</classpath>
diff --git a/hyracks-examples/btree-example/btreehelper/.project b/hyracks-examples/btree-example/btreehelper/.project
new file mode 100644
index 0000000..c65470b
--- /dev/null
+++ b/hyracks-examples/btree-example/btreehelper/.project
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>btreehelper</name>
+ <comment></comment>
+ <projects>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ <buildCommand>
+ <name>org.maven.ide.eclipse.maven2Builder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ <nature>org.maven.ide.eclipse.maven2Nature</nature>
+ </natures>
+</projectDescription>
diff --git a/hyracks-examples/btree-example/btreehelper/.settings/org.eclipse.jdt.core.prefs b/hyracks-examples/btree-example/btreehelper/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 0000000..367ddc4
--- /dev/null
+++ b/hyracks-examples/btree-example/btreehelper/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,6 @@
+#Wed Oct 06 08:06:49 PDT 2010
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
+org.eclipse.jdt.core.compiler.compliance=1.6
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-examples/btree-example/btreehelper/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/btree-example/btreehelper/.settings/org.maven.ide.eclipse.prefs
new file mode 100644
index 0000000..6733f8d
--- /dev/null
+++ b/hyracks-examples/btree-example/btreehelper/.settings/org.maven.ide.eclipse.prefs
@@ -0,0 +1,9 @@
+#Wed Oct 06 08:06:48 PDT 2010
+activeProfiles=
+eclipse.preferences.version=1
+fullBuildGoals=process-test-resources
+includeModules=false
+resolveWorkspaceProjects=true
+resourceFilterGoals=process-resources resources\:testResources
+skipCompilerPlugin=true
+version=1
diff --git a/hyracks-examples/hyracks-integration-tests/.classpath b/hyracks-examples/hyracks-integration-tests/.classpath
index f2cc5f7..b93a5df 100644
--- a/hyracks-examples/hyracks-integration-tests/.classpath
+++ b/hyracks-examples/hyracks-integration-tests/.classpath
@@ -3,5 +3,5 @@
<classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
+ <classpathentry kind="output" path="target/android-classes"/>
</classpath>
diff --git a/hyracks-examples/text-example/textapp/.classpath b/hyracks-examples/text-example/textapp/.classpath
index 3f62785..1a0f9bf 100644
--- a/hyracks-examples/text-example/textapp/.classpath
+++ b/hyracks-examples/text-example/textapp/.classpath
@@ -2,5 +2,5 @@
<classpath>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.4"/>
<classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
+ <classpathentry kind="output" path="target/android-classes"/>
</classpath>
diff --git a/hyracks-examples/text-example/textclient/.classpath b/hyracks-examples/text-example/textclient/.classpath
index 1f3c1ff..9b2abd7 100644
--- a/hyracks-examples/text-example/textclient/.classpath
+++ b/hyracks-examples/text-example/textclient/.classpath
@@ -3,5 +3,5 @@
<classpathentry kind="src" output="target/classes" path="src/main/java"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
+ <classpathentry kind="output" path="target/android-classes"/>
</classpath>
diff --git a/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java b/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java
index b7a5639..b255258 100644
--- a/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java
+++ b/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java
@@ -37,6 +37,7 @@
import edu.uci.ics.hyracks.dataflow.common.data.hash.UTF8StringBinaryHashFunctionFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.normalizers.UTF8StringNormalizedKeyComputerFactory;
import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
import edu.uci.ics.hyracks.dataflow.std.aggregators.CountAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.aggregators.IFieldValueResultingAggregatorFactory;
@@ -51,6 +52,7 @@
import edu.uci.ics.hyracks.dataflow.std.group.HashGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.sort.InMemorySortOperatorDescriptor;
import edu.uci.ics.hyracks.examples.text.WordTupleParserFactory;
public class WordCountMain {
@@ -128,30 +130,32 @@
UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
IOperatorDescriptor gBy;
+ int[] keys = new int[] { 0 };
if ("hash".equalsIgnoreCase(algo)) {
- gBy = new HashGroupOperatorDescriptor(spec, new int[] { 0 },
- new FieldHashPartitionComputerFactory(new int[] { 0 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }),
+ gBy = new HashGroupOperatorDescriptor(spec, keys, new FieldHashPartitionComputerFactory(keys,
+ new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }),
new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
new MultiAggregatorFactory(
new IFieldValueResultingAggregatorFactory[] { new CountAggregatorFactory() }),
groupResultDesc, htSize);
gBy.setPartitionConstraint(createPartitionConstraint(outSplits));
IConnectorDescriptor scanGroupConn = new MToNHashPartitioningConnectorDescriptor(spec,
- new FieldHashPartitionComputerFactory(new int[] { 0 },
+ new FieldHashPartitionComputerFactory(keys,
new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
spec.connect(scanGroupConn, wordScanner, 0, gBy, 0);
} else {
- ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, sbSize, new int[] { 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, wordDesc);
+ IBinaryComparatorFactory[] cfs = new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE };
+ IOperatorDescriptor sorter = "memsort".equalsIgnoreCase(algo) ? new InMemorySortOperatorDescriptor(spec,
+ keys, new UTF8StringNormalizedKeyComputerFactory(), cfs, wordDesc)
+ : new ExternalSortOperatorDescriptor(spec, sbSize, keys, cfs, wordDesc);
sorter.setPartitionConstraint(createPartitionConstraint(outSplits));
IConnectorDescriptor scanSortConn = new MToNHashPartitioningConnectorDescriptor(spec,
- new FieldHashPartitionComputerFactory(new int[] { 0 },
+ new FieldHashPartitionComputerFactory(keys,
new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
spec.connect(scanSortConn, wordScanner, 0, sorter, 0);
- gBy = new PreclusteredGroupOperatorDescriptor(spec, new int[] { 0 },
+ gBy = new PreclusteredGroupOperatorDescriptor(spec, keys,
new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
new MultiAggregatorFactory(
new IFieldValueResultingAggregatorFactory[] { new CountAggregatorFactory() }),
diff --git a/hyracks-examples/text-example/texthelper/.classpath b/hyracks-examples/text-example/texthelper/.classpath
index 1f3c1ff..9b2abd7 100644
--- a/hyracks-examples/text-example/texthelper/.classpath
+++ b/hyracks-examples/text-example/texthelper/.classpath
@@ -3,5 +3,5 @@
<classpathentry kind="src" output="target/classes" path="src/main/java"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
+ <classpathentry kind="output" path="target/android-classes"/>
</classpath>
diff --git a/hyracks-examples/tpch-example/tpchapp/.classpath b/hyracks-examples/tpch-example/tpchapp/.classpath
index 3f62785..1a0f9bf 100644
--- a/hyracks-examples/tpch-example/tpchapp/.classpath
+++ b/hyracks-examples/tpch-example/tpchapp/.classpath
@@ -2,5 +2,5 @@
<classpath>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.4"/>
<classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
+ <classpathentry kind="output" path="target/android-classes"/>
</classpath>
diff --git a/hyracks-examples/tpch-example/tpchclient/.classpath b/hyracks-examples/tpch-example/tpchclient/.classpath
index 1f3c1ff..9b2abd7 100644
--- a/hyracks-examples/tpch-example/tpchclient/.classpath
+++ b/hyracks-examples/tpch-example/tpchclient/.classpath
@@ -3,5 +3,5 @@
<classpathentry kind="src" output="target/classes" path="src/main/java"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
+ <classpathentry kind="output" path="target/android-classes"/>
</classpath>