Merged hyracks_asterix_stabilization upto rev 1913
git-svn-id: https://hyracks.googlecode.com/svn/trunk@1924 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-common/pom.xml b/hyracks/hyracks-algebricks/hyracks-algebricks-common/pom.xml
new file mode 100644
index 0000000..5996eda
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-common/pom.xml
@@ -0,0 +1,31 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-algebricks-common</artifactId>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-api</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksAbsolutePartitionConstraint.java b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksAbsolutePartitionConstraint.java
similarity index 94%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksAbsolutePartitionConstraint.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksAbsolutePartitionConstraint.java
index de5f778..924d85b 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksAbsolutePartitionConstraint.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksAbsolutePartitionConstraint.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.api.constraints;
+package edu.uci.ics.hyracks.algebricks.common.constraints;
public class AlgebricksAbsolutePartitionConstraint extends AlgebricksPartitionConstraint {
private final String[] locations;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksCountPartitionConstraint.java b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksCountPartitionConstraint.java
similarity index 94%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksCountPartitionConstraint.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksCountPartitionConstraint.java
index 9cd8079..99091aa 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksCountPartitionConstraint.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksCountPartitionConstraint.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.api.constraints;
+package edu.uci.ics.hyracks.algebricks.common.constraints;
public class AlgebricksCountPartitionConstraint extends AlgebricksPartitionConstraint {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksPartitionConstraint.java b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraint.java
similarity index 92%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksPartitionConstraint.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraint.java
index 0a63b80..929479c 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksPartitionConstraint.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraint.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.api.constraints;
+package edu.uci.ics.hyracks.algebricks.common.constraints;
public abstract class AlgebricksPartitionConstraint {
public enum PartitionConstraintType {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksPartitionConstraintHelper.java b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraintHelper.java
similarity index 97%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksPartitionConstraintHelper.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraintHelper.java
index e64602b..54cdb59 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksPartitionConstraintHelper.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraintHelper.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.api.constraints;
+package edu.uci.ics.hyracks.algebricks.common.constraints;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/exceptions/AlgebricksException.java b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/exceptions/AlgebricksException.java
similarity index 94%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/exceptions/AlgebricksException.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/exceptions/AlgebricksException.java
index c852ecd..b55a4b3 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/exceptions/AlgebricksException.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/exceptions/AlgebricksException.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.api.exceptions;
+package edu.uci.ics.hyracks.algebricks.common.exceptions;
public class AlgebricksException extends Exception {
private static final long serialVersionUID = 1L;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/exceptions/NotImplementedException.java b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/exceptions/NotImplementedException.java
similarity index 94%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/exceptions/NotImplementedException.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/exceptions/NotImplementedException.java
index f5343a8..cb9e054 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/exceptions/NotImplementedException.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/exceptions/NotImplementedException.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.api.exceptions;
+package edu.uci.ics.hyracks.algebricks.common.exceptions;
public class NotImplementedException extends RuntimeException {
private static final long serialVersionUID = 2L;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/ListSet.java b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/ListSet.java
new file mode 100644
index 0000000..acf1e6f
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/ListSet.java
@@ -0,0 +1,111 @@
+package edu.uci.ics.hyracks.algebricks.common.utils;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+public class ListSet<E> implements Set<E> {
+ private List<E> list = new ArrayList<E>();
+
+ public ListSet() {
+ }
+
+ public ListSet(Collection<? extends E> arg) {
+ this.addAll(arg);
+ }
+
+ @Override
+ public boolean add(E arg0) {
+ if (list.contains(arg0))
+ return false;
+ return list.add(arg0);
+ }
+
+ @Override
+ public boolean addAll(Collection<? extends E> arg0) {
+ for (E item : arg0)
+ if (list.contains(item))
+ return false;
+ return list.addAll(arg0);
+ }
+
+ @Override
+ public void clear() {
+ list.clear();
+ }
+
+ @Override
+ public boolean contains(Object arg0) {
+ return list.contains(arg0);
+ }
+
+ @Override
+ public boolean containsAll(Collection<?> arg0) {
+ return list.containsAll(arg0);
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return list.isEmpty();
+ }
+
+ @Override
+ public Iterator<E> iterator() {
+ return list.iterator();
+ }
+
+ @Override
+ public boolean remove(Object arg0) {
+ return list.remove(arg0);
+ }
+
+ @Override
+ public boolean removeAll(Collection<?> arg0) {
+ return list.removeAll(arg0);
+ }
+
+ @Override
+ public boolean retainAll(Collection<?> arg0) {
+ return list.retainAll(arg0);
+ }
+
+ @Override
+ public int size() {
+ return list.size();
+ }
+
+ @Override
+ public Object[] toArray() {
+ return list.toArray();
+ }
+
+ @Override
+ public <T> T[] toArray(T[] arg0) {
+ return list.toArray(arg0);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public boolean equals(Object arg) {
+ if (!(arg instanceof Set))
+ return false;
+ Set<E> set = (Set<E>) arg;
+ for (E item : set) {
+ if (!this.contains(item))
+ return false;
+ }
+ for (E item : this) {
+ if (!set.contains(item))
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return list.toString();
+ }
+
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Pair.java b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/Pair.java
similarity index 96%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Pair.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/Pair.java
index d3aff9f..ddd56da 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Pair.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/Pair.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.utils;
+package edu.uci.ics.hyracks.algebricks.common.utils;
import java.io.Serializable;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Triple.java b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/Triple.java
similarity index 96%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Triple.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/Triple.java
index ad76ef1..0361152 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Triple.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/Triple.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.utils;
+package edu.uci.ics.hyracks.algebricks.common.utils;
public class Triple<T1, T2, T3> {
public T1 first;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/AbstractCompilerFactoryBuilder.java b/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/AbstractCompilerFactoryBuilder.java
index 276398d..f1e7acb 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/AbstractCompilerFactoryBuilder.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/AbstractCompilerFactoryBuilder.java
@@ -16,25 +16,25 @@
import java.util.List;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryBooleanInspector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryHashFunctionFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryIntegerInspector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.INormalizedKeyComputerFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.ISerializerDeserializerProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.ITypeTraitProvider;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IPartialAggregationTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspectorFactory;
+import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.ISerializerDeserializerProvider;
+import edu.uci.ics.hyracks.algebricks.data.ITypeTraitProvider;
import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
public abstract class AbstractCompilerFactoryBuilder {
@@ -45,10 +45,10 @@
protected ISerializerDeserializerProvider serializerDeserializerProvider;
protected IBinaryHashFunctionFactoryProvider hashFunctionFactoryProvider;
protected IBinaryComparatorFactoryProvider comparatorFactoryProvider;
- protected IBinaryBooleanInspector binaryBooleanInspector;
- protected IBinaryIntegerInspector binaryIntegerInspector;
+ protected IBinaryBooleanInspectorFactory binaryBooleanInspectorFactory;
+ protected IBinaryIntegerInspectorFactory binaryIntegerInspectorFactory;
protected IPrinterFactoryProvider printerProvider;
- protected ILogicalExpressionJobGen exprJobGen;
+ protected IExpressionRuntimeProvider expressionRuntimeProvider;
protected IExpressionTypeComputer expressionTypeComputer;
protected INullableTypeComputer nullableTypeComputer;
protected IExpressionEvalSizeComputer expressionEvalSizeComputer;
@@ -102,20 +102,20 @@
return comparatorFactoryProvider;
}
- public void setBinaryBooleanInspector(IBinaryBooleanInspector binaryBooleanInspector) {
- this.binaryBooleanInspector = binaryBooleanInspector;
+ public void setBinaryBooleanInspectorFactory(IBinaryBooleanInspectorFactory binaryBooleanInspectorFactory) {
+ this.binaryBooleanInspectorFactory = binaryBooleanInspectorFactory;
}
- public IBinaryBooleanInspector getBinaryBooleanInspector() {
- return binaryBooleanInspector;
+ public IBinaryBooleanInspectorFactory getBinaryBooleanInspectorFactory() {
+ return binaryBooleanInspectorFactory;
}
- public void setBinaryIntegerInspector(IBinaryIntegerInspector binaryIntegerInspector) {
- this.binaryIntegerInspector = binaryIntegerInspector;
+ public void setBinaryIntegerInspectorFactory(IBinaryIntegerInspectorFactory binaryIntegerInspectorFactory) {
+ this.binaryIntegerInspectorFactory = binaryIntegerInspectorFactory;
}
- public IBinaryIntegerInspector getBinaryIntegerInspector() {
- return binaryIntegerInspector;
+ public IBinaryIntegerInspectorFactory getBinaryIntegerInspectorFactory() {
+ return binaryIntegerInspectorFactory;
}
public void setPrinterProvider(IPrinterFactoryProvider printerProvider) {
@@ -126,12 +126,12 @@
return printerProvider;
}
- public void setExprJobGen(ILogicalExpressionJobGen exprJobGen) {
- this.exprJobGen = exprJobGen;
+ public void setExpressionRuntimeProvider(IExpressionRuntimeProvider expressionRuntimeProvider) {
+ this.expressionRuntimeProvider = expressionRuntimeProvider;
}
- public ILogicalExpressionJobGen getExprJobGen() {
- return exprJobGen;
+ public IExpressionRuntimeProvider getExpressionRuntimeProvider() {
+ return expressionRuntimeProvider;
}
public void setExpressionTypeComputer(IExpressionTypeComputer expressionTypeComputer) {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java b/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
index 9245772..1d21463 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
@@ -14,6 +14,7 @@
*/
package edu.uci.ics.hyracks.algebricks.compiler.api;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
@@ -21,10 +22,9 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.PlanCompiler;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.PlanCompiler;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AlgebricksOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.HeuristicOptimizer;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IOptimizationContextFactory;
@@ -85,10 +85,10 @@
AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Starting Job Generation.\n");
JobGenContext context = new JobGenContext(null, metadata, appContext,
serializerDeserializerProvider, hashFunctionFactoryProvider, comparatorFactoryProvider,
- typeTraitProvider, binaryBooleanInspector, binaryIntegerInspector, printerProvider,
- nullWriterFactory, normalizedKeyComputerFactoryProvider, exprJobGen,
- expressionTypeComputer, nullableTypeComputer, oc, expressionEvalSizeComputer,
- partialAggregationTypeComputer, frameSize, clusterLocations);
+ typeTraitProvider, binaryBooleanInspectorFactory, binaryIntegerInspectorFactory,
+ printerProvider, nullWriterFactory, normalizedKeyComputerFactoryProvider,
+ expressionRuntimeProvider, expressionTypeComputer, nullableTypeComputer, oc,
+ expressionEvalSizeComputer, partialAggregationTypeComputer, frameSize, clusterLocations);
PlanCompiler pc = new PlanCompiler(context);
return pc.compilePlan(plan, null);
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompiler.java b/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompiler.java
index 654337a..9146722 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompiler.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompiler.java
@@ -14,7 +14,7 @@
*/
package edu.uci.ics.hyracks.algebricks.compiler.api;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.api.job.JobSpecification;
public interface ICompiler {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java b/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java
index 129ec26..8ba6583 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java
@@ -18,8 +18,8 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java b/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java
index 8be65f4..978a647 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java
@@ -18,19 +18,16 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
/**
- *
* Runs rules sequentially (round-robin), until one iteration over all rules
* produces no change.
*
- *
* @author Nicola
- *
*/
public class SequentialFixpointRuleController extends AbstractRuleController {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java b/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java
index 5cf62e0..1e37d14 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java
@@ -18,8 +18,8 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/pom.xml b/hyracks/hyracks-algebricks/hyracks-algebricks-core/pom.xml
index 9fb84a7..f3e9b74 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/pom.xml
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/pom.xml
@@ -37,5 +37,15 @@
<artifactId>hyracks-dataflow-std</artifactId>
<version>0.2.1-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks-runtime</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks-common</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </dependency>
</dependencies>
</project>
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IHyracksJobBuilder.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IHyracksJobBuilder.java
index 19c85c6..4c6e747 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IHyracksJobBuilder.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IHyracksJobBuilder.java
@@ -16,9 +16,9 @@
import java.util.List;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalExpression.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalExpression.java
index a54a88d..b1a2408 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalExpression.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalExpression.java
@@ -20,9 +20,9 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public interface ILogicalExpression {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java
index 719b5df..165fccd 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java
@@ -19,16 +19,16 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
public interface ILogicalOperator {
@@ -89,8 +89,4 @@
public IPhysicalPropertiesVector getDeliveredPhysicalProperties();
public void computeDeliveredPhysicalProperties(IOptimizationContext context) throws AlgebricksException;
-
- public void setHostQueryContext(Object context);
-
- public Object getHostQueryContext();
}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlanAndMetadata.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlanAndMetadata.java
index 5a616c6..1d354de 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlanAndMetadata.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlanAndMetadata.java
@@ -14,8 +14,8 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
public interface ILogicalPlanAndMetadata {
public ILogicalPlan getPlan();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IOptimizationContext.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
index 4daaceb..468d25c 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
@@ -17,6 +17,7 @@
import java.util.List;
import java.util.Map;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableEvalSizeEnvironment;
@@ -24,7 +25,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java
index 46d2dc3..58c2870 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java
@@ -14,11 +14,11 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
public interface IPhysicalOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
index 01fa9da..5234d2c 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
@@ -44,5 +44,6 @@
WRITE_RESULT,
INSERT_DELETE,
INDEX_INSERT_DELETE,
- UPDATE
+ UPDATE,
+ EXTENSION_OPERATOR
}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
index 48f320b..a969372 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
@@ -5,6 +5,7 @@
ASSIGN,
BROADCAST_EXCHANGE,
BTREE_SEARCH,
+ STATS,
DATASOURCE_SCAN,
EMPTY_TUPLE_SOURCE,
EXTERNAL_GROUP_BY,
@@ -41,5 +42,7 @@
WRITE_RESULT,
INSERT_DELETE,
INDEX_INSERT_DELETE,
- UPDATE
+ UPDATE,
+ INVERTED_INDEX_SEARCH,
+ PARTITIONINGSPLIT
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IAWriter.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IAWriter.java
deleted file mode 100644
index d29fab6..0000000
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IAWriter.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
-
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-
-public interface IAWriter {
-
- public void init() throws AlgebricksException;
-
- public void printTuple(FrameTupleAccessor tAccess, int tIdx) throws AlgebricksException;
-}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java
index dae055c..0d9e763 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java
@@ -146,11 +146,11 @@
public void getConstraintsAndEquivClasses(Collection<FunctionalDependency> fds,
Map<LogicalVariable, EquivalenceClass> equivClasses) {
FunctionIdentifier funId = getFunctionIdentifier();
- if (funId == AlgebricksBuiltinFunctions.AND) {
+ if (funId.equals(AlgebricksBuiltinFunctions.AND)) {
for (Mutable<ILogicalExpression> a : arguments) {
a.getValue().getConstraintsAndEquivClasses(fds, equivClasses);
}
- } else if (funId == AlgebricksBuiltinFunctions.EQ) {
+ } else if (funId.equals(AlgebricksBuiltinFunctions.EQ)) {
ILogicalExpression opLeft = arguments.get(0).getValue();
ILogicalExpression opRight = arguments.get(1).getValue();
if (opLeft.getExpressionTag() == LogicalExpressionTag.CONSTANT
@@ -170,11 +170,11 @@
@Override
public void getConstraintsForOuterJoin(Collection<FunctionalDependency> fds, Collection<LogicalVariable> outerVars) {
FunctionIdentifier funId = getFunctionIdentifier();
- if (funId == AlgebricksBuiltinFunctions.AND) {
+ if (funId.equals(AlgebricksBuiltinFunctions.AND)) {
for (Mutable<ILogicalExpression> a : arguments) {
a.getValue().getConstraintsForOuterJoin(fds, outerVars);
}
- } else if (funId == AlgebricksBuiltinFunctions.EQ) {
+ } else if (funId.equals(AlgebricksBuiltinFunctions.EQ)) {
ILogicalExpression opLeft = arguments.get(0).getValue();
ILogicalExpression opRight = arguments.get(1).getValue();
if (opLeft.getExpressionTag() == LogicalExpressionTag.VARIABLE
@@ -221,7 +221,7 @@
@Override
public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs) {
- if (getFunctionIdentifier() != AlgebricksBuiltinFunctions.AND || arguments.size() <= 1) {
+ if (!getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.AND) || arguments.size() <= 1) {
return false;
} else {
conjs.addAll(arguments);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java
index ac77ee2..a720039 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java
@@ -18,10 +18,10 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
/**
*
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
index 2fa0637..a0b7674 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
@@ -21,11 +21,11 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public final class ConstantExpression extends AbstractLogicalExpression {
private IAlgebricksConstantValue value;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionEvalSizeComputer.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionEvalSizeComputer.java
index 8dd807e..623f7bb 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionEvalSizeComputer.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionEvalSizeComputer.java
@@ -14,8 +14,8 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public interface IExpressionEvalSizeComputer {
// size in bytes, or -1 if unknown
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionRuntimeProvider.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionRuntimeProvider.java
new file mode 100644
index 0000000..5db1a0d
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionRuntimeProvider.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory;
+
+public interface IExpressionRuntimeProvider {
+ public IScalarEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
+ IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException;
+
+ public IAggregateEvaluatorFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr,
+ IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+ throws AlgebricksException;
+
+ public ICopySerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory(
+ AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas,
+ JobGenContext context) throws AlgebricksException;
+
+ public IRunningAggregateEvaluatorFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr,
+ IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+ throws AlgebricksException;
+
+ public IUnnestingEvaluatorFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr,
+ IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+ throws AlgebricksException;
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionTypeComputer.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionTypeComputer.java
index 72861ae..3057d8a 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionTypeComputer.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionTypeComputer.java
@@ -14,9 +14,9 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public interface IExpressionTypeComputer {
public Object getType(ILogicalExpression expr, IMetadataProvider<?, ?> metadataProvider,
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ILogicalExpressionJobGen.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ILogicalExpressionJobGen.java
index babfede..8d46662 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ILogicalExpressionJobGen.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ILogicalExpressionJobGen.java
@@ -14,34 +14,34 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IRunningAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.ISerializableAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IUnnestingFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyRunningAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyUnnestingFunctionFactory;
public interface ILogicalExpressionJobGen {
- public IEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
+ public ICopyEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException;
- public IAggregateFunctionFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr,
+ public ICopyAggregateFunctionFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr,
IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
throws AlgebricksException;
- public ISerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory(
+ public ICopySerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory(
AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas,
JobGenContext context) throws AlgebricksException;
- public IRunningAggregateFunctionFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr,
+ public ICopyRunningAggregateFunctionFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr,
IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
throws AlgebricksException;
- public IUnnestingFunctionFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr,
+ public ICopyUnnestingFunctionFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr,
IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
throws AlgebricksException;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IMergeAggregationExpressionFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IMergeAggregationExpressionFactory.java
index 36a414c..04bb824 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IMergeAggregationExpressionFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IMergeAggregationExpressionFactory.java
@@ -1,8 +1,8 @@
package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public interface IMergeAggregationExpressionFactory {
ILogicalExpression createMergeAggregation(ILogicalExpression expr, IOptimizationContext env)
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java
index 3290e35..37b6d80 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java
@@ -1,6 +1,6 @@
package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
public interface INullableTypeComputer {
public Object makeNullableType(Object type) throws AlgebricksException;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IPartialAggregationTypeComputer.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IPartialAggregationTypeComputer.java
index 88e34e9..5921335 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IPartialAggregationTypeComputer.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IPartialAggregationTypeComputer.java
@@ -1,8 +1,8 @@
package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public interface IPartialAggregationTypeComputer {
public Object getType(ILogicalExpression expr, IVariableTypeEnvironment env,
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java
index 0827b97..2275227 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java
@@ -16,9 +16,9 @@
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public interface IVariableTypeEnvironment {
public Object getVarType(LogicalVariable var) throws AlgebricksException;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IndexedNLJoinExpressionAnnotation.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IndexedNLJoinExpressionAnnotation.java
new file mode 100644
index 0000000..709bfa1
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IndexedNLJoinExpressionAnnotation.java
@@ -0,0 +1,26 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+public class IndexedNLJoinExpressionAnnotation implements IExpressionAnnotation {
+
+ public static final String INDEXED_NL_JOIN_ANNOTATION_KEY = "indexnl";
+ public static final IndexedNLJoinExpressionAnnotation INSTANCE = new IndexedNLJoinExpressionAnnotation();
+
+ private Object object;
+
+ @Override
+ public Object getObject() {
+ return object;
+ }
+
+ @Override
+ public void setObject(Object object) {
+ this.object = object;
+ }
+
+ @Override
+ public IExpressionAnnotation copy() {
+ IndexedNLJoinExpressionAnnotation clone = new IndexedNLJoinExpressionAnnotation();
+ clone.setObject(object);
+ return clone;
+ }
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/LogicalExpressionJobGenToExpressionRuntimeProviderAdapter.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/LogicalExpressionJobGenToExpressionRuntimeProviderAdapter.java
new file mode 100644
index 0000000..91c9eae
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/LogicalExpressionJobGenToExpressionRuntimeProviderAdapter.java
@@ -0,0 +1,193 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyAggregateFunction;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyRunningAggregateFunction;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyRunningAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyUnnestingFunction;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyUnnestingFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class LogicalExpressionJobGenToExpressionRuntimeProviderAdapter implements IExpressionRuntimeProvider {
+ private final ILogicalExpressionJobGen lejg;
+
+ public LogicalExpressionJobGenToExpressionRuntimeProviderAdapter(ILogicalExpressionJobGen lejg) {
+ this.lejg = lejg;
+ }
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
+ IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException {
+ ICopyEvaluatorFactory cef = lejg.createEvaluatorFactory(expr, env, inputSchemas, context);
+ return new ScalarEvaluatorFactoryAdapter(cef);
+ }
+
+ @Override
+ public IAggregateEvaluatorFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr,
+ IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+ throws AlgebricksException {
+ ICopyAggregateFunctionFactory caff = lejg.createAggregateFunctionFactory(expr, env, inputSchemas, context);
+ return new AggregateFunctionFactoryAdapter(caff);
+ }
+
+ @Override
+ public ICopySerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory(
+ AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas,
+ JobGenContext context) throws AlgebricksException {
+ return lejg.createSerializableAggregateFunctionFactory(expr, env, inputSchemas, context);
+ }
+
+ @Override
+ public IRunningAggregateEvaluatorFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr,
+ IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+ throws AlgebricksException {
+ ICopyRunningAggregateFunctionFactory craff = lejg.createRunningAggregateFunctionFactory(expr, env,
+ inputSchemas, context);
+ return new RunningAggregateFunctionFactoryAdapter(craff);
+ }
+
+ @Override
+ public IUnnestingEvaluatorFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr,
+ IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+ throws AlgebricksException {
+ ICopyUnnestingFunctionFactory cuff = lejg.createUnnestingFunctionFactory(expr, env, inputSchemas, context);
+ return new UnnestingFunctionFactoryAdapter(cuff);
+ }
+
+ public static final class ScalarEvaluatorFactoryAdapter implements IScalarEvaluatorFactory {
+ private static final long serialVersionUID = 1L;
+
+ private final ICopyEvaluatorFactory cef;
+
+ public ScalarEvaluatorFactoryAdapter(ICopyEvaluatorFactory cef) {
+ this.cef = cef;
+ }
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
+ final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ final ICopyEvaluator ce = cef.createEvaluator(abvs);
+ return new IScalarEvaluator() {
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws AlgebricksException {
+ abvs.reset();
+ ce.evaluate(tuple);
+ result.set(abvs);
+ }
+ };
+ }
+ }
+
+ public static final class AggregateFunctionFactoryAdapter implements IAggregateEvaluatorFactory {
+ private static final long serialVersionUID = 1L;
+
+ private final ICopyAggregateFunctionFactory caff;
+
+ public AggregateFunctionFactoryAdapter(ICopyAggregateFunctionFactory caff) {
+ this.caff = caff;
+ }
+
+ @Override
+ public IAggregateEvaluator createAggregateEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
+ final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ final ICopyAggregateFunction caf = caff.createAggregateFunction(abvs);
+ return new IAggregateEvaluator() {
+ @Override
+ public void step(IFrameTupleReference tuple) throws AlgebricksException {
+ caf.step(tuple);
+ }
+
+ @Override
+ public void init() throws AlgebricksException {
+ abvs.reset();
+ caf.init();
+ }
+
+ @Override
+ public void finish(IPointable result) throws AlgebricksException {
+ caf.finish();
+ result.set(abvs);
+ }
+ };
+ }
+ }
+
+ public static final class RunningAggregateFunctionFactoryAdapter implements IRunningAggregateEvaluatorFactory {
+ private static final long serialVersionUID = 1L;
+
+ private final ICopyRunningAggregateFunctionFactory craff;
+
+ public RunningAggregateFunctionFactoryAdapter(ICopyRunningAggregateFunctionFactory craff) {
+ this.craff = craff;
+ }
+
+ @Override
+ public IRunningAggregateEvaluator createRunningAggregateEvaluator() throws AlgebricksException {
+ final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ final ICopyRunningAggregateFunction craf = craff.createRunningAggregateFunction(abvs);
+ return new IRunningAggregateEvaluator() {
+ @Override
+ public void step(IFrameTupleReference tuple, IPointable result) throws AlgebricksException {
+ abvs.reset();
+ craf.step(tuple);
+ result.set(abvs);
+ }
+
+ @Override
+ public void init() throws AlgebricksException {
+ craf.init();
+ }
+ };
+ }
+ }
+
+ public static final class UnnestingFunctionFactoryAdapter implements IUnnestingEvaluatorFactory {
+ private static final long serialVersionUID = 1L;
+
+ private final ICopyUnnestingFunctionFactory cuff;
+
+ public UnnestingFunctionFactoryAdapter(ICopyUnnestingFunctionFactory cuff) {
+ this.cuff = cuff;
+ }
+
+ @Override
+ public IUnnestingEvaluator createUnnestingEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
+ final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ final ICopyUnnestingFunction cuf = cuff.createUnnestingFunction(abvs);
+ return new IUnnestingEvaluator() {
+ @Override
+ public boolean step(IPointable result) throws AlgebricksException {
+ abvs.reset();
+ if (cuf.step()) {
+ result.set(abvs);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void init(IFrameTupleReference tuple) throws AlgebricksException {
+ cuf.init(tuple);
+ }
+ };
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java
index fa8838b..bf3f82b 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java
@@ -18,10 +18,10 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class ScalarFunctionCallExpression extends AbstractFunctionCallExpression {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java
index c5cd897..40f25b2 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java
@@ -18,12 +18,12 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPropertiesComputer;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class StatefulFunctionCallExpression extends AbstractFunctionCallExpression {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java
index 5f3de4b..71932d8 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java
@@ -18,10 +18,10 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class UnnestingFunctionCallExpression extends AbstractFunctionCallExpression {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java
index b21eba3..76a7ec6 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java
@@ -19,11 +19,11 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class VariableReferenceExpression extends AbstractLogicalExpression {
private int tupleRef;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/AlgebricksBuiltinFunctions.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/AlgebricksBuiltinFunctions.java
index 7511155..de9cf29 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/AlgebricksBuiltinFunctions.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/AlgebricksBuiltinFunctions.java
@@ -14,7 +14,6 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.functions;
-import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
@@ -28,69 +27,27 @@
NEQ
}
- private static Map<FunctionIdentifier, FunctionIdentifier> algebricksBuiltinFunctions = new HashMap<FunctionIdentifier, FunctionIdentifier>();
-
- private static Map<FunctionIdentifier, IFunctionInfo> _finfos = new HashMap<FunctionIdentifier, IFunctionInfo>();
-
public static final String ALGEBRICKS_NS = "algebricks";
// comparisons
- public final static FunctionIdentifier EQ = new FunctionIdentifier(ALGEBRICKS_NS, "eq", 2, true);
- public final static FunctionIdentifier LE = new FunctionIdentifier(ALGEBRICKS_NS, "le", 2, true);
- public final static FunctionIdentifier GE = new FunctionIdentifier(ALGEBRICKS_NS, "ge", 2, true);
- public final static FunctionIdentifier LT = new FunctionIdentifier(ALGEBRICKS_NS, "lt", 2, true);
- public final static FunctionIdentifier GT = new FunctionIdentifier(ALGEBRICKS_NS, "gt", 2, true);
- public final static FunctionIdentifier NEQ = new FunctionIdentifier(ALGEBRICKS_NS, "neq", 2, true);
+ public final static FunctionIdentifier EQ = new FunctionIdentifier(ALGEBRICKS_NS, "eq", 2);
+ public final static FunctionIdentifier LE = new FunctionIdentifier(ALGEBRICKS_NS, "le", 2);
+ public final static FunctionIdentifier GE = new FunctionIdentifier(ALGEBRICKS_NS, "ge", 2);
+ public final static FunctionIdentifier LT = new FunctionIdentifier(ALGEBRICKS_NS, "lt", 2);
+ public final static FunctionIdentifier GT = new FunctionIdentifier(ALGEBRICKS_NS, "gt", 2);
+ public final static FunctionIdentifier NEQ = new FunctionIdentifier(ALGEBRICKS_NS, "neq", 2);
// booleans
- public final static FunctionIdentifier NOT = new FunctionIdentifier(ALGEBRICKS_NS, "not", 1, true);
+ public final static FunctionIdentifier NOT = new FunctionIdentifier(ALGEBRICKS_NS, "not", 1);
public final static FunctionIdentifier AND = new FunctionIdentifier(ALGEBRICKS_NS, "and",
- FunctionIdentifier.VARARGS, true);
- public final static FunctionIdentifier OR = new FunctionIdentifier(ALGEBRICKS_NS, "or", FunctionIdentifier.VARARGS,
- true);
+ FunctionIdentifier.VARARGS);
+ public final static FunctionIdentifier OR = new FunctionIdentifier(ALGEBRICKS_NS, "or", FunctionIdentifier.VARARGS);
// numerics
- public final static FunctionIdentifier NUMERIC_ADD = new FunctionIdentifier(ALGEBRICKS_NS, "numeric-add", 2, true);
+ public final static FunctionIdentifier NUMERIC_ADD = new FunctionIdentifier(ALGEBRICKS_NS, "numeric-add", 2);
// nulls
- public final static FunctionIdentifier IS_NULL = new FunctionIdentifier(ALGEBRICKS_NS, "is-null", 1, true);
-
- static {
- // comparisons
- add(EQ);
- add(LE);
- add(GE);
- add(LT);
- add(GT);
- add(NEQ);
- // booleans
- add(NOT);
- add(AND);
- add(OR);
- // numerics
- add(NUMERIC_ADD);
- // nulls
- add(IS_NULL);
- for (FunctionIdentifier fi : algebricksBuiltinFunctions.values()) {
- _finfos.put(fi, new FunctionInfoImpl(fi));
- }
- }
-
- private static void add(FunctionIdentifier fi) {
- algebricksBuiltinFunctions.put(fi, fi);
- }
-
- public static final boolean isAlgebricksBuiltinFunction(FunctionIdentifier fi) {
- return algebricksBuiltinFunctions.get(fi) != null;
- }
-
- public static final Collection<FunctionIdentifier> getAlgebricksBuiltinFunctions() {
- return algebricksBuiltinFunctions.values();
- }
-
- public static final FunctionIdentifier getBuiltinFunctionIdentifier(FunctionIdentifier fi) {
- return algebricksBuiltinFunctions.get(fi);
- }
+ public final static FunctionIdentifier IS_NULL = new FunctionIdentifier(ALGEBRICKS_NS, "is-null", 1);
private static final Map<FunctionIdentifier, ComparisonKind> comparisonFunctions = new HashMap<FunctionIdentifier, ComparisonKind>();
static {
@@ -109,9 +66,4 @@
public static boolean isComparisonFunction(FunctionIdentifier fi) {
return comparisonFunctions.get(fi) != null;
}
-
- public static IFunctionInfo getBuiltinFunctionInfo(FunctionIdentifier fi) {
- return _finfos.get(fi);
- }
-
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java
index 252f23c..548f646 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java
@@ -14,36 +14,31 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.functions;
-public final class FunctionIdentifier {
- final private String namespace;
- final private String name;
- final private boolean isBuiltin;
- final private int arity;
+import java.io.Serializable;
+
+public final class FunctionIdentifier implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final String namespace;
+ private final String name;
+ private final int arity;
public final static int VARARGS = -1;
- public FunctionIdentifier(String namespace, String name, boolean isBuiltin) {
- this.namespace = namespace;
- this.name = name;
- this.arity = VARARGS;
- this.isBuiltin = isBuiltin;
+ public FunctionIdentifier(String namespace, String name) {
+ this(namespace, name, VARARGS);
}
- public FunctionIdentifier(String namespace, String name, int arity, boolean isBuiltin) {
+ public FunctionIdentifier(String namespace, String name, int arity) {
this.namespace = namespace;
this.name = name;
this.arity = arity;
- this.isBuiltin = isBuiltin;
}
public String getName() {
return name;
}
- public boolean isBuiltin() {
- return isBuiltin;
- }
-
@Override
public boolean equals(Object o) {
if (super.equals(o)) {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java
index 57b7262..96afc8e 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java
@@ -14,8 +14,7 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.functions;
+
public interface IFunctionInfo {
FunctionIdentifier getFunctionIdentifier();
-
- Object getInfo();
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
index 221992e..899b633 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
@@ -16,14 +16,18 @@
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.job.JobSpecification;
@@ -38,7 +42,8 @@
*/
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<S> dataSource,
List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
- JobGenContext context, JobSpecification jobSpec) throws AlgebricksException;
+ IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec)
+ throws AlgebricksException;
public boolean scannerOperatorIsLeaf(IDataSource<S> dataSource);
@@ -58,15 +63,77 @@
IOperatorSchema propagatedSchema, List<LogicalVariable> keys, LogicalVariable payLoadVar,
RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec) throws AlgebricksException;
+ /**
+ * Creates the insert runtime of IndexInsertDeletePOperator, which models
+ * insert/delete operations into a secondary index.
+ *
+ * @param dataSource
+ * Target secondary index.
+ * @param propagatedSchema
+ * Output schema of the insert/delete operator to be created.
+ * @param inputSchemas
+ * Output schemas of the insert/delete operator to be created.
+ * @param typeEnv
+ * Type environment of the original IndexInsertDeleteOperator operator.
+ * @param primaryKeys
+ * Variables for the dataset's primary keys that the dataSource secondary index belongs to.
+ * @param secondaryKeys
+ * Variables for the secondary-index keys.
+ * @param filterExpr
+ * Filtering expression to be pushed inside the runtime op.
+ * Such a filter may, e.g., exclude NULLs from being inserted/deleted.
+ * @param recordDesc
+ * Output record descriptor of the runtime op to be created.
+ * @param context
+ * Job generation context.
+ * @param spec
+ * Target job specification.
+ * @return
+ * A Hyracks IOperatorDescriptor and its partition constraint.
+ * @throws AlgebricksException
+ */
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexInsertRuntime(
- IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, List<LogicalVariable> primaryKeys,
- List<LogicalVariable> secondaryKeys, RecordDescriptor recordDesc, JobGenContext context,
- JobSpecification spec) throws AlgebricksException;
+ IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
+ IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
+ ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec)
+ throws AlgebricksException;
+ /**
+ * Creates the delete runtime of IndexInsertDeletePOperator, which models
+ * insert/delete operations into a secondary index.
+ *
+ * @param dataSource
+ * Target secondary index.
+ * @param propagatedSchema
+ * Output schema of the insert/delete operator to be created.
+ * @param inputSchemas
+ * Output schemas of the insert/delete operator to be created.
+ * @param typeEnv
+ * Type environment of the original IndexInsertDeleteOperator operator.
+ * @param primaryKeys
+ * Variables for the dataset's primary keys that the dataSource secondary index belongs to.
+ * @param secondaryKeys
+ * Variables for the secondary-index keys.
+ * @param filterExpr
+ * Filtering expression to be pushed inside the runtime op.
+ * Such a filter may, e.g., exclude NULLs from being inserted/deleted.
+ * @param recordDesc
+ * Output record descriptor of the runtime op to be created.
+ * @param context
+ * Job generation context.
+ * @param spec
+ * Target job specification.
+ * @return
+ * A Hyracks IOperatorDescriptor and its partition constraint.
+ * @throws AlgebricksException
+ */
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexDeleteRuntime(
- IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, List<LogicalVariable> primaryKeys,
- List<LogicalVariable> secondaryKeys, RecordDescriptor recordDesc, JobGenContext context,
- JobSpecification spec) throws AlgebricksException;
+ IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
+ IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
+ ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec)
+ throws AlgebricksException;
public IDataSourceIndex<I, S> findDataSourceIndex(I indexId, S dataSourceId) throws AlgebricksException;
+
+ public IFunctionInfo lookupFunction(FunctionIdentifier fid);
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java
index 456010a..0e83902 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java
@@ -19,14 +19,13 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
/**
* @author Nicola
- *
*/
public abstract class AbstractAssignOperator extends AbstractLogicalOperator {
protected final List<LogicalVariable> variables;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java
index c2911d4..5ab44a4 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java
@@ -18,12 +18,12 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public abstract class AbstractBinaryJoinOperator extends AbstractLogicalOperator {
protected final Mutable<ILogicalExpression> condition;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractExtensibleLogicalOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractExtensibleLogicalOperator.java
new file mode 100644
index 0000000..10ecb1e
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractExtensibleLogicalOperator.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+
+/**
+ * @author rico
+ */
+public abstract class AbstractExtensibleLogicalOperator implements IOperatorExtension {
+
+ private AbstractLogicalOperator.ExecutionMode mode = AbstractLogicalOperator.ExecutionMode.UNPARTITIONED;
+ protected List<LogicalVariable> schema;
+ protected IPhysicalOperator physicalOperator;
+
+ @Override
+ public ExecutionMode getExecutionMode() {
+ return mode;
+ }
+
+ @Override
+ public void setExecutionMode(ExecutionMode mode) {
+ this.mode = mode;
+ }
+
+ @Override
+ public void setSchema(List<LogicalVariable> schema) {
+ this.schema = schema;
+ }
+
+ @Override
+ public IPhysicalOperator getPhysicalOperator() {
+ return physicalOperator;
+ }
+
+ @Override
+ public void setPhysicalOperator(IPhysicalOperator physicalOperator) {
+ this.physicalOperator = physicalOperator;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
index 88fd0d5..dc0edfe 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
@@ -21,6 +21,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -31,12 +32,11 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
public abstract class AbstractLogicalOperator implements ILogicalOperator {
@@ -59,7 +59,6 @@
protected IPhysicalOperator physicalOperator;
private final Map<String, Object> annotations = new HashMap<String, Object>();
private boolean bJobGenEnabled = true;
- private Object hostQueryContext;
final protected List<Mutable<ILogicalOperator>> inputs;
// protected List<LogicalOperatorReference> outputs;
@@ -168,14 +167,6 @@
return bJobGenEnabled;
}
- public void setHostQueryContext(Object context) {
- this.hostQueryContext = context;
- }
-
- public Object getHostQueryContext() {
- return hostQueryContext;
- }
-
@Override
public IVariableTypeEnvironment computeInputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
return createPropagatingAllInputsTypeEnvironment(ctx);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
index e4e7ec8..4d8cce0 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
@@ -17,9 +17,9 @@
import java.util.ArrayList;
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public abstract class AbstractScanOperator extends AbstractLogicalOperator {
protected List<LogicalVariable> variables;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java
index 22f73ff..9eaf059 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java
@@ -18,10 +18,10 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public abstract class AbstractUnnestOperator extends AbstractScanOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java
index 1555a76..2f53f9b 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java
@@ -5,6 +5,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -13,13 +14,13 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class AggregateOperator extends AbstractAssignOperator {
// private ArrayList<AggregateFunctionCallExpression> expressions;
// TODO type safe list of expressions
private List<Mutable<ILogicalExpression>> mergeExpressions;
+ private LogicalVariable partitioningVariable;
public AggregateOperator(List<LogicalVariable> variables, List<Mutable<ILogicalExpression>> expressions) {
super(variables, expressions);
@@ -68,6 +69,14 @@
return mergeExpressions;
}
+ public void setPartitioningVariable(LogicalVariable partitioningVariable) {
+ this.partitioningVariable = partitioningVariable;
+ }
+
+ public LogicalVariable getPartitioningVariable() {
+ return partitioningVariable;
+ }
+
@Override
public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java
index eedd3a2..a4dc2e0 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java
@@ -18,6 +18,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -25,14 +26,11 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
/**
- *
* It corresponds to the Map operator in other algebras.
*
* @author Nicola
- *
*/
public class AssignOperator extends AbstractAssignOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
index bcc1192..3227f3d 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
@@ -18,6 +18,7 @@
import java.util.Collection;
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -26,7 +27,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class DataSourceScanOperator extends AbstractScanOperator {
private IDataSource<?> dataSource;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DieOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DieOperator.java
index 2e330cf..20aa574 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DieOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DieOperator.java
@@ -19,6 +19,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -27,7 +28,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class DieOperator extends AbstractLogicalOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java
index 4d479e1..ee0dcf6 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java
@@ -19,6 +19,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
@@ -29,7 +30,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class DistinctOperator extends AbstractLogicalOperator {
private final List<Mutable<ILogicalExpression>> expressions;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java
index 732dfbe..45f9909 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java
@@ -17,6 +17,7 @@
import java.util.ArrayList;
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -25,7 +26,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class EmptyTupleSourceOperator extends AbstractLogicalOperator {
@@ -56,7 +56,7 @@
return visitor.visitEmptyTupleSourceOperator(this, arg);
}
- @Override
+ @Override
public void recomputeSchema() {
schema = new ArrayList<LogicalVariable>();
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java
index 3e046b5..aba5f83 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java
@@ -17,6 +17,7 @@
import java.util.ArrayList;
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -25,7 +26,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class ExchangeOperator extends AbstractLogicalOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java
new file mode 100644
index 0000000..101b6f5
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+/**
+ * @author rico
+ */
+public class ExtensionOperator extends AbstractLogicalOperator {
+
+ private IOperatorExtension delegate;
+
+ public ExtensionOperator(IOperatorExtension delegate) {
+ super();
+ if (delegate == null) {
+ throw new IllegalArgumentException("delegate cannot be null!");
+ }
+ this.delegate = delegate;
+ setExecutionMode(delegate.getExecutionMode());
+ }
+
+ @Override
+ public void recomputeSchema() throws AlgebricksException {
+ schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
+ delegate.setSchema(schema);
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitExtensionOperator(this, arg);
+ }
+
+ @Override
+ public boolean isMap() {
+ return this.delegate.isMap();
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.ALL;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return this.createPropagatingAllInputsTypeEnvironment(ctx);
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.EXTENSION_OPERATOR;
+ }
+
+ public IOperatorExtension getNewInstanceOfDelegateOperator() {
+ return delegate.newInstance();
+ }
+
+ @Override
+ public List<LogicalVariable> getSchema() {
+ return this.schema;
+ }
+
+ @Override
+ public ExecutionMode getExecutionMode() {
+ return delegate.getExecutionMode();
+ }
+
+ @Override
+ public void setExecutionMode(ExecutionMode mode) {
+ delegate.setExecutionMode(mode);
+ }
+
+ @Override
+ public IPhysicalOperator getPhysicalOperator() {
+ return delegate.getPhysicalOperator();
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeInputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return this.createPropagatingAllInputsTypeEnvironment(ctx);
+ }
+
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java
index 762f034..ba396f4 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java
@@ -21,6 +21,8 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -37,8 +39,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
public class GroupByOperator extends AbstractOperatorWithNestedPlans {
// If the LogicalVariable in a pair is null, it means that the GroupBy is
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IOperatorExtension.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IOperatorExtension.java
new file mode 100644
index 0000000..98c3301
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IOperatorExtension.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+
+/**
+ * @author rico
+ */
+public interface IOperatorExtension {
+
+ void setExecutionMode(ExecutionMode mode);
+
+ boolean isMap();
+
+ public IOperatorExtension newInstance();
+
+ boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException;
+
+ void setSchema(List<LogicalVariable> schema);
+
+ IPhysicalOperator getPhysicalOperator();
+
+ void setPhysicalOperator(IPhysicalOperator physicalOperator);
+
+ ExecutionMode getExecutionMode();
+
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
index 7264763..becde1c 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
@@ -5,6 +5,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -15,21 +16,22 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class IndexInsertDeleteOperator extends AbstractLogicalOperator {
private final IDataSourceIndex<?, ?> dataSourceIndex;
private final List<Mutable<ILogicalExpression>> primaryKeyExprs;
private final List<Mutable<ILogicalExpression>> secondaryKeyExprs;
+ private final Mutable<ILogicalExpression> filterExpr;
private final Kind operation;
public IndexInsertDeleteOperator(IDataSourceIndex<?, ?> dataSourceIndex,
List<Mutable<ILogicalExpression>> primaryKeyExprs, List<Mutable<ILogicalExpression>> secondaryKeyExprs,
- Kind operation) {
+ Mutable<ILogicalExpression> filterExpr, Kind operation) {
this.dataSourceIndex = dataSourceIndex;
this.primaryKeyExprs = primaryKeyExprs;
this.secondaryKeyExprs = secondaryKeyExprs;
+ this.filterExpr = filterExpr;
this.operation = operation;
}
@@ -92,6 +94,10 @@
return secondaryKeyExprs;
}
+ public Mutable<ILogicalExpression> getFilterExpression() {
+ return filterExpr;
+ }
+
public Kind getOperation() {
return operation;
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java
index c2f0161..54fab7c 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java
@@ -16,13 +16,13 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class InnerJoinOperator extends AbstractBinaryJoinOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
index 6699228..91e51f6 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
@@ -5,6 +5,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -14,7 +15,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class InsertDeleteOperator extends AbstractLogicalOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java
index 7c5be7e..60da2cf 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java
@@ -16,6 +16,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
@@ -26,7 +27,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class LeftOuterJoinOperator extends AbstractBinaryJoinOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java
index 21738f6..3c6a699 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java
@@ -19,6 +19,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -27,7 +28,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class LimitOperator extends AbstractLogicalOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java
index 189a410..a9a8f55 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java
@@ -18,6 +18,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -29,7 +30,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class NestedTupleSourceOperator extends AbstractLogicalOperator {
private final Mutable<ILogicalOperator> dataSourceReference;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java
index 3a5cdde..812c219 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java
@@ -19,6 +19,8 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -28,8 +30,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
public class OrderOperator extends AbstractLogicalOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java
index a76237b..6fe6bed 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java
@@ -15,9 +15,11 @@
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -26,26 +28,48 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.PartitioningSplitOperatorDescriptor;
+/**
+ * Partitions it's input based on a given list of expressions.
+ * Each expression is assumed to return true/false,
+ * and there is exactly one output branch per expression (optionally, plus one default branch).
+ * For each input tuple, the expressions are evaluated one-by-one,
+ * and the tuple is written to first output branch whose corresponding
+ * expression evaluates to true.
+ * If all expressions evaluate to false, then
+ * the tuple is written to the default output branch, if any was given.
+ * If no output branch was given, then such tuples are simply dropped.
+ * Given N expressions there may be N or N+1 output branches because the default output branch may be separate from the regular output branches.
+ */
public class PartitioningSplitOperator extends AbstractLogicalOperator {
- private Mutable<ILogicalExpression>[] expressions;
- private boolean hasDefault;
+ private final List<Mutable<ILogicalExpression>> expressions;
+ private final int defaultBranchIndex;
- public PartitioningSplitOperator(Mutable<ILogicalExpression>[] exprList, boolean defaultBranchDefined) {
- expressions = exprList;
- hasDefault = defaultBranchDefined;
+ public PartitioningSplitOperator(List<Mutable<ILogicalExpression>> expressions, int defaultBranchIndex) throws AlgebricksException {
+ this.expressions = expressions;
+ this.defaultBranchIndex = defaultBranchIndex;
+ // Check that the default output branch index is in [0, N], where N is the number of expressions.
+ if (defaultBranchIndex != PartitioningSplitOperatorDescriptor.NO_DEFAULT_BRANCH
+ && defaultBranchIndex > expressions.size()) {
+ throw new AlgebricksException("Default branch index out of bounds. Number of exprs given: "
+ + expressions.size() + ". The maximum default branch index may therefore be: " + expressions.size());
+ }
}
- public Mutable<ILogicalExpression>[] getExpressions() {
+ public List<Mutable<ILogicalExpression>> getExpressions() {
return expressions;
}
- public boolean hasDefault() {
- return hasDefault;
+ public int getDefaultBranchIndex() {
+ return defaultBranchIndex;
}
-
+
+ public int getNumOutputBranches() {
+ return (defaultBranchIndex == expressions.size()) ? expressions.size() + 1 : expressions.size();
+ }
+
@Override
public LogicalOperatorTag getOperatorTag() {
return LogicalOperatorTag.PARTITIONINGSPLIT;
@@ -65,8 +89,8 @@
@Override
public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
boolean b = false;
- for (int i = 0; i < expressions.length; i++) {
- if (visitor.transform(expressions[i])) {
+ for (int i = 0; i < expressions.size(); i++) {
+ if (visitor.transform(expressions.get(i))) {
b = true;
}
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java
index 736ae1d..459e5dd 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java
@@ -17,6 +17,7 @@
import java.util.ArrayList;
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -25,7 +26,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class ProjectOperator extends AbstractLogicalOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java
index f594373..a4e9d2b 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java
@@ -16,6 +16,7 @@
import java.util.ArrayList;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -23,7 +24,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class ReplicateOperator extends AbstractLogicalOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
index 798597e..3694503 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
@@ -18,6 +18,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -25,7 +26,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class RunningAggregateOperator extends AbstractAssignOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java
index dc2795c..c75fb40 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java
@@ -16,6 +16,8 @@
import java.util.ArrayList;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -25,8 +27,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
public class ScriptOperator extends AbstractLogicalOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java
index c1e038e..8c611b0 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java
@@ -18,6 +18,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
@@ -34,7 +35,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class SelectOperator extends AbstractLogicalOperator {
private final Mutable<ILogicalExpression> condition;
@@ -85,11 +85,11 @@
ctx.getNullableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers);
if (condition.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
AbstractFunctionCallExpression f1 = (AbstractFunctionCallExpression) condition.getValue();
- if (f1.getFunctionIdentifier() == AlgebricksBuiltinFunctions.NOT) {
+ if (f1.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.NOT)) {
ILogicalExpression a1 = f1.getArguments().get(0).getValue();
if (a1.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
AbstractFunctionCallExpression f2 = (AbstractFunctionCallExpression) a1;
- if (f2.getFunctionIdentifier() == AlgebricksBuiltinFunctions.IS_NULL) {
+ if (f2.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.IS_NULL)) {
ILogicalExpression a2 = f2.getArguments().get(0).getValue();
if (a2.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
LogicalVariable var = ((VariableReferenceExpression) a2).getVariableReference();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java
index a75a9a9..1b12423 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java
@@ -2,6 +2,7 @@
import java.util.ArrayList;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -13,7 +14,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class SinkOperator extends AbstractLogicalOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java
index 0d1704b..89707eb 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java
@@ -21,6 +21,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
@@ -35,7 +36,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class SubplanOperator extends AbstractOperatorWithNestedPlans {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java
index cec7d6f..c7100dd 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java
@@ -17,6 +17,8 @@
import java.util.ArrayList;
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -25,8 +27,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
public class UnionAllOperator extends AbstractLogicalOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
index c4f5cb0..b9a137f 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
@@ -18,6 +18,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -26,22 +27,22 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
/**
* This operator may go away after we add indexes to Algebricks.
- *
*/
public class UnnestMapOperator extends AbstractUnnestOperator {
private final List<Object> variableTypes; // TODO: get rid of this and
-
+ private final boolean propagateInput;
+
// deprecate UnnestMap
public UnnestMapOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression,
- List<Object> variableTypes) {
+ List<Object> variableTypes, boolean propagateInput) {
super(variables, expression);
this.variableTypes = variableTypes;
+ this.propagateInput = propagateInput;
}
@Override
@@ -55,19 +56,19 @@
}
/**
- *
* UnnestMap doesn't propagate input variables, because currently it is only
* used to search indexes. In the future, it would be nice to have the
* choice to propagate input variables or not.
- *
*/
@Override
public VariablePropagationPolicy getVariablePropagationPolicy() {
return new VariablePropagationPolicy() {
-
@Override
public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
throws AlgebricksException {
+ if (propagateInput) {
+ target.addAllVariables(sources[0]);
+ }
for (LogicalVariable v : variables) {
target.addVariable(v);
}
@@ -81,13 +82,28 @@
@Override
public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
- IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
- ctx.getMetadataProvider());
+ IVariableTypeEnvironment env = null;
+ if (propagateInput) {
+ env = createPropagatingAllInputsTypeEnvironment(ctx);
+ } else {
+ env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getMetadataProvider());
+ }
int n = variables.size();
for (int i = 0; i < n; i++) {
env.setVarType(variables.get(i), variableTypes.get(i));
}
return env;
}
+
+ public boolean propagatesInput() {
+ return propagateInput;
+ }
+
+ /*
+ @Override
+ public boolean isMap() {
+ return !propagateInput;
+ }
+ */
}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java
index f85cabb..2e1ef16 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java
@@ -18,13 +18,13 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class UnnestOperator extends AbstractUnnestOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java
index 4f94f34..f156490 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java
@@ -1,12 +1,12 @@
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class UpdateOperator extends AbstractLogicalOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java
index fdcdfd8..4c14333 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java
@@ -19,6 +19,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -28,7 +29,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class WriteOperator extends AbstractLogicalOperator {
private List<Mutable<ILogicalExpression>> expressions;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
index 6d05c76..48065a5 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
@@ -19,6 +19,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -28,7 +29,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class WriteResultOperator extends AbstractLogicalOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
index 0c0f257..0539cbe 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
@@ -25,6 +25,9 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
@@ -61,6 +64,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -70,10 +74,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
public class FDsAndEquivClassesVisitor implements ILogicalOperatorVisitor<Void, IOptimizationContext> {
@@ -637,4 +638,10 @@
return null;
}
+ @Override
+ public Void visitExtensionOperator(ExtensionOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ propagateFDsAndEquivClasses(op, ctx);
+ return null;
+ }
+
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
index 5d9e6b9..31061db 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
@@ -15,7 +15,6 @@
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -24,6 +23,9 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -38,6 +40,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -64,9 +67,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
public class IsomorphismOperatorVisitor implements ILogicalOperatorVisitor<Boolean, ILogicalOperator> {
@@ -110,6 +110,14 @@
}
@Override
+ public Boolean visitExtensionOperator(ExtensionOperator op, ILogicalOperator arg) throws AlgebricksException {
+ ExtensionOperator aop = (ExtensionOperator) copyAndSubstituteVar(op, arg);
+ if (aop.getOperatorTag() != LogicalOperatorTag.EXTENSION_OPERATOR)
+ return Boolean.FALSE;
+ return Boolean.TRUE;
+ }
+
+ @Override
public Boolean visitGroupByOperator(GroupByOperator op, ILogicalOperator arg) throws AlgebricksException {
AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
// require the same physical operator, otherwise delivers different data
@@ -259,8 +267,7 @@
if (aop.getOperatorTag() != LogicalOperatorTag.PARTITIONINGSPLIT)
return Boolean.FALSE;
PartitioningSplitOperator partitionOpArg = (PartitioningSplitOperator) copyAndSubstituteVar(op, arg);
- boolean isomorphic = compareExpressions(Arrays.asList(op.getExpressions()),
- Arrays.asList(partitionOpArg.getExpressions()));
+ boolean isomorphic = compareExpressions(op.getExpressions(), partitionOpArg.getExpressions());
return isomorphic;
}
@@ -677,9 +684,8 @@
public ILogicalOperator visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg)
throws AlgebricksException {
ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
- deepCopyExpressionRefs(newExpressions, Arrays.asList(op.getExpressions()));
- return new PartitioningSplitOperator(newExpressions.toArray(new Mutable[0]),
- op.hasDefault());
+ deepCopyExpressionRefs(newExpressions, op.getExpressions());
+ return new PartitioningSplitOperator(newExpressions, op.getDefaultBranchIndex());
}
@Override
@@ -726,7 +732,7 @@
ArrayList<LogicalVariable> newInputList = new ArrayList<LogicalVariable>();
newInputList.addAll(op.getVariables());
return new UnnestMapOperator(newInputList, deepCopyExpressionRef(op.getExpressionRef()),
- new ArrayList<Object>(op.getVariableTypes()));
+ new ArrayList<Object>(op.getVariableTypes()), op.propagatesInput());
}
@Override
@@ -778,8 +784,10 @@
deepCopyExpressionRefs(newPrimaryKeyExpressions, op.getPrimaryKeyExpressions());
List<Mutable<ILogicalExpression>> newSecondaryKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
deepCopyExpressionRefs(newSecondaryKeyExpressions, op.getSecondaryKeyExpressions());
+ Mutable<ILogicalExpression> newFilterExpression = new MutableObject<ILogicalExpression>(((AbstractLogicalExpression)op.getFilterExpression())
+ .cloneExpression());
return new IndexInsertDeleteOperator(op.getDataSourceIndex(), newPrimaryKeyExpressions,
- newSecondaryKeyExpressions, op.getOperation());
+ newSecondaryKeyExpressions, newFilterExpression, op.getOperation());
}
@Override
@@ -807,6 +815,11 @@
deepCopyExpressionRef(pair.second)));
return newOrdersAndExprs;
}
+
+ @Override
+ public ILogicalOperator visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
+ return new ExtensionOperator(op.getNewInstanceOfDelegateOperator());
+ }
}
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismUtilities.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismUtilities.java
index 21271fd..02533c3 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismUtilities.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismUtilities.java
@@ -16,9 +16,9 @@
import java.util.Map;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class IsomorphismUtilities {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
index 340b855..562bb4c 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
@@ -22,6 +22,9 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -51,6 +54,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -58,9 +62,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
public class IsomorphismVariableMappingVisitor implements ILogicalOperatorVisitor<Void, ILogicalOperator> {
@@ -414,4 +415,10 @@
return false;
}
+ @Override
+ public Void visitExtensionOperator(ExtensionOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
index 30ad3ab..9b2f5a0 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
@@ -16,6 +16,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -45,6 +46,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -54,7 +56,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LogicalPropertiesVectorImpl;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
public class LogicalPropertiesVisitor implements ILogicalOperatorVisitor<Void, IOptimizationContext> {
@@ -318,4 +319,10 @@
}
}
+ @Override
+ public Void visitExtensionOperator(ExtensionOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
index cd05624..994c6cb 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
@@ -20,6 +20,9 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -46,6 +49,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -53,9 +57,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
public class ProducedVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
@@ -245,4 +246,9 @@
public Void visitSinkOperator(SinkOperator op, Void arg) throws AlgebricksException {
return null;
}
+
+ @Override
+ public Void visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
index 6aa3796..9295179 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
@@ -18,6 +18,8 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -46,6 +48,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -53,8 +56,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
public class SchemaVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
@@ -267,4 +268,10 @@
return null;
}
+ @Override
+ public Void visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
index b85c798..11e56ca 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
@@ -19,6 +19,9 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -48,6 +51,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -57,9 +61,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
public class SubstituteVariableVisitor implements ILogicalOperatorVisitor<Void, Pair<LogicalVariable, LogicalVariable>> {
@@ -409,4 +410,10 @@
IVariableTypeEnvironment env = ctx.getOutputTypeEnvironment(op);
env.substituteProducedVariable(arg.first, arg.second);
}
+
+ @Override
+ public Void visitExtensionOperator(ExtensionOperator op, Pair<LogicalVariable, LogicalVariable> arg)
+ throws AlgebricksException {
+ return null;
+ }
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
index d9d6a6a..3a82ccd 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
@@ -19,6 +19,9 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -46,6 +49,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -53,9 +57,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
public class UsedVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
@@ -70,6 +71,9 @@
for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
exprRef.getValue().getUsedVariables(usedVariables);
}
+ if (op.getPartitioningVariable() != null) {
+ usedVariables.add(op.getPartitioningVariable());
+ }
return null;
}
@@ -291,4 +295,9 @@
return null;
}
+ @Override
+ public Void visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java
index 8e9c5b8..25f22e7 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java
@@ -21,12 +21,12 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
public class VariableUtilities {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java
index 761c7f2..0aa9cd4 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java
@@ -14,15 +14,15 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
public abstract class AbstractExchangePOperator extends AbstractPhysicalOperator {
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
@@ -40,6 +40,7 @@
return false;
}
- public abstract Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(JobSpecification spec,
- ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException;
+ public abstract Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(
+ IConnectorDescriptorRegistry spec, ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context)
+ throws AlgebricksException;
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java
index b9352f3..f188265 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java
@@ -14,11 +14,14 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -33,9 +36,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
public abstract class AbstractHashJoinPOperator extends AbstractJoinPOperator {
@@ -75,7 +75,7 @@
pp = pv0.getPartitioningProperty();
}
} else {
- pp = null;
+ pp = IPartitioningProperty.UNPARTITIONED;
}
this.deliveredProperties = new StructuralPropertiesVector(pp, deliveredLocalProperties(iop, context));
}
@@ -93,8 +93,8 @@
if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
switch (partitioningType) {
case PAIRWISE: {
- pp1 = new UnorderedPartitionedProperty(new HashSet<LogicalVariable>(keysLeftBranch), null);
- pp2 = new UnorderedPartitionedProperty(new HashSet<LogicalVariable>(keysRightBranch), null);
+ pp1 = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(keysLeftBranch), null);
+ pp2 = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(keysRightBranch), null);
break;
}
case BROADCAST: {
@@ -131,9 +131,9 @@
UnorderedPartitionedProperty upp1 = (UnorderedPartitionedProperty) firstDeliveredPartitioning;
Set<LogicalVariable> set1 = upp1.getColumnSet();
UnorderedPartitionedProperty uppreq = (UnorderedPartitionedProperty) requirements;
- Set<LogicalVariable> modifuppreq = new HashSet<LogicalVariable>();
+ Set<LogicalVariable> modifuppreq = new ListSet<LogicalVariable>();
Map<LogicalVariable, EquivalenceClass> eqmap = context.getEquivalenceClassMap(op);
- Set<LogicalVariable> covered = new HashSet<LogicalVariable>();
+ Set<LogicalVariable> covered = new ListSet<LogicalVariable>();
for (LogicalVariable r : uppreq.getColumnSet()) {
EquivalenceClass ecSnd = eqmap.get(r);
boolean found = false;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java
index 620c161..c5f4c71 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java
@@ -16,6 +16,10 @@
import java.util.Map;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -28,14 +32,10 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.PlanCompiler;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksCountPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.PlanCompiler;
+import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
index 84982aa..7dd6372 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
@@ -2,13 +2,14 @@
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
@@ -35,7 +36,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PropertiesUtil;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
public abstract class AbstractPreclusteredGroupByPOperator extends AbstractPhysicalOperator {
@@ -74,7 +74,7 @@
switch (lsp.getPropertyType()) {
case LOCAL_GROUPING_PROPERTY: {
LocalGroupingProperty lgp = (LocalGroupingProperty) lsp;
- Set<LogicalVariable> colSet = new HashSet<LogicalVariable>();
+ Set<LogicalVariable> colSet = new ListSet<LogicalVariable>();
for (LogicalVariable v : lgp.getColumnSet()) {
LogicalVariable v2 = getLhsGbyVar(gby, v);
if (v2 != null) {
@@ -118,7 +118,7 @@
List<ILocalStructuralProperty> localProps = null;
localProps = new ArrayList<ILocalStructuralProperty>(1);
- Set<LogicalVariable> gbvars = new HashSet<LogicalVariable>(columnList);
+ Set<LogicalVariable> gbvars = new ListSet<LogicalVariable>(columnList);
LocalGroupingProperty groupProp = new LocalGroupingProperty(gbvars, new ArrayList<LogicalVariable>(columnList));
GroupByOperator gby = (GroupByOperator) op;
@@ -194,7 +194,7 @@
IPartitioningProperty pp = null;
AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
- pp = new UnorderedPartitionedProperty(new HashSet<LogicalVariable>(columnList), null);
+ pp = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(columnList), null);
}
pv[0] = new StructuralPropertiesVector(pp, localProps);
return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
index ca09b77..63ea8b9 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
@@ -19,6 +19,8 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -37,8 +39,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
public abstract class AbstractStableSortPOperator extends AbstractPhysicalOperator {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java
index 47038ad..81dc2c2 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java
@@ -16,9 +16,12 @@
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
@@ -26,19 +29,21 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg.AggregateRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.AggregateRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
public class AggregatePOperator extends AbstractPhysicalOperator {
@@ -62,7 +67,16 @@
@Override
public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
IPhysicalPropertiesVector reqdByParent) {
- return emptyUnaryRequirements();
+ AggregateOperator aggOp = (AggregateOperator) op;
+ if (aggOp.getExecutionMode() == ExecutionMode.PARTITIONED && aggOp.getPartitioningVariable() != null) {
+ StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
+ Set<LogicalVariable> partitioningVariables = new ListSet<LogicalVariable>();
+ partitioningVariables.add(aggOp.getPartitioningVariable());
+ pv[0] = new StructuralPropertiesVector(new UnorderedPartitionedProperty(partitioningVariables, null), null);
+ return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ } else {
+ return emptyUnaryRequirements();
+ }
}
@Override
@@ -76,19 +90,18 @@
for (int i = 0; i < outColumns.length; i++) {
outColumns[i] = opSchema.findVariable(variables.get(i));
}
- IAggregateFunctionFactory[] aggFactories = new IAggregateFunctionFactory[expressions.size()];
- ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
+ IAggregateEvaluatorFactory[] aggFactories = new IAggregateEvaluatorFactory[expressions.size()];
+ IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
for (int i = 0; i < aggFactories.length; i++) {
- AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) expressions.get(i)
- .getValue();
- aggFactories[i] = exprJobGen.createAggregateFunctionFactory(aggFun,
+ AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) expressions.get(i).getValue();
+ aggFactories[i] = expressionRuntimeProvider.createAggregateFunctionFactory(aggFun,
context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
}
AggregateRuntimeFactory runtime = new AggregateRuntimeFactory(aggFactories);
// contribute one Asterix framewriter
- RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
builder.contributeMicroOperator(aggOp, runtime, recDesc);
// and contribute one edge from its child
ILogicalOperator src = aggOp.getInputs().get(0).getValue();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
index 861765d..f5bd9d9 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
@@ -18,22 +18,22 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.AssignRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.AssignRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
public class AssignPOperator extends AbstractPhysicalOperator {
@@ -66,10 +66,10 @@
for (int i = 0; i < outColumns.length; i++) {
outColumns[i] = opSchema.findVariable(variables.get(i));
}
- IEvaluatorFactory[] evalFactories = new IEvaluatorFactory[expressions.size()];
- ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
+ IScalarEvaluatorFactory[] evalFactories = new IScalarEvaluatorFactory[expressions.size()];
+ IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
for (int i = 0; i < evalFactories.length; i++) {
- evalFactories[i] = exprJobGen.createEvaluatorFactory(expressions.get(i).getValue(),
+ evalFactories[i] = expressionRuntimeProvider.createEvaluatorFactory(expressions.get(i).getValue(),
context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
}
@@ -79,7 +79,7 @@
AssignRuntimeFactory runtime = new AssignRuntimeFactory(outColumns, evalFactories, projectionList);
// contribute one Asterix framewriter
- RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
builder.contributeMicroOperator(assign, runtime, recDesc);
// and contribute one edge from its child
ILogicalOperator src = assign.getInputs().get(0).getValue();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java
index 0c780d5..bf19342 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java
@@ -14,6 +14,8 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -26,11 +28,9 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.connectors.MToNReplicatingConnectorDescriptor;
public class BroadcastPOperator extends AbstractExchangePOperator {
@@ -61,7 +61,7 @@
}
@Override
- public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(JobSpecification spec,
+ public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
IConnectorDescriptor conn = new MToNReplicatingConnectorDescriptor(spec);
return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
index 68a6a24..a2dac3f 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
@@ -16,20 +16,21 @@
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
@SuppressWarnings("rawtypes")
@@ -66,11 +67,11 @@
throws AlgebricksException {
DataSourceScanOperator scan = (DataSourceScanOperator) op;
IMetadataProvider mp = context.getMetadataProvider();
-
+ IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
List<LogicalVariable> vars = scan.getVariables();
List<LogicalVariable> projectVars = scan.getProjectVariables();
Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> p = mp.getScannerRuntime(dataSource, vars,
- projectVars, scan.isProjectPushed(), context, builder.getJobSpec());
+ projectVars, scan.isProjectPushed(), opSchema, typeEnv, context, builder.getJobSpec());
builder.contributeHyracksOperator(scan, p.first);
if (p.second != null) {
builder.contributeAlgebricksPartitionConstraint(p.first, p.second);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java
index 7fc76da..bf42c30 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java
@@ -16,6 +16,7 @@
import java.util.LinkedList;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -26,9 +27,8 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.EmptyTupleSourceRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.EmptyTupleSourceRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java
index f13b2bb..55add37 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java
@@ -1,12 +1,15 @@
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
import java.util.ArrayList;
-import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
+import java.util.Set;
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
@@ -16,7 +19,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IPartialAggregationTypeComputer;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
@@ -32,23 +35,21 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.ISerializableAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.OperatorSchemaImpl;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg.SerializableAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.OperatorSchemaImpl;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.SerializableAggregatorDescriptorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.ExternalGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.group.HashSpillableTableFactory;
import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.external.ExternalGroupOperatorDescriptor;
public class ExternalGroupByPOperator extends AbstractPhysicalOperator {
@@ -98,7 +99,7 @@
List<ILocalStructuralProperty> propsLocal = new LinkedList<ILocalStructuralProperty>();
GroupByOperator gOp = (GroupByOperator) op;
- HashSet<LogicalVariable> columnSet = new HashSet<LogicalVariable>();
+ Set<LogicalVariable> columnSet = new ListSet<LogicalVariable>();
if (!columnSet.isEmpty()) {
propsLocal.add(new LocalGroupingProperty(columnSet));
@@ -121,7 +122,7 @@
AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
- pv[0] = new StructuralPropertiesVector(new UnorderedPartitionedProperty(new HashSet<LogicalVariable>(
+ pv[0] = new StructuralPropertiesVector(new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(
columnSet), null), null);
return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
} else {
@@ -166,15 +167,15 @@
IPartialAggregationTypeComputer partialAggregationTypeComputer = context.getPartialAggregationTypeComputer();
List<Object> intermediateTypes = new ArrayList<Object>();
int n = aggOp.getExpressions().size();
- ISerializableAggregateFunctionFactory[] aff = new ISerializableAggregateFunctionFactory[n];
+ ICopySerializableAggregateFunctionFactory[] aff = new ICopySerializableAggregateFunctionFactory[n];
int i = 0;
- ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
+ IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
IVariableTypeEnvironment aggOpInputEnv = context.getTypeEnvironment(aggOp.getInputs().get(0).getValue());
IVariableTypeEnvironment outputEnv = context.getTypeEnvironment(op);
for (Mutable<ILogicalExpression> exprRef : aggOp.getExpressions()) {
AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) exprRef.getValue();
- aff[i++] = exprJobGen.createSerializableAggregateFunctionFactory(aggFun, aggOpInputEnv, inputSchemas,
- context);
+ aff[i++] = expressionRuntimeProvider.createSerializableAggregateFunctionFactory(aggFun, aggOpInputEnv,
+ inputSchemas, context);
intermediateTypes.add(partialAggregationTypeComputer.getType(aggFun, aggOpInputEnv,
context.getMetadataProvider()));
}
@@ -197,14 +198,14 @@
aggOpInputEnv.setVarType(var, outputEnv.getVarType(var));
compileSubplans(inputSchemas[0], gby, opSchema, context);
- JobSpecification spec = builder.getJobSpec();
+ IOperatorDescriptorRegistry spec = builder.getJobSpec();
IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(gbyCols,
aggOpInputEnv, context);
- RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
IBinaryHashFunctionFactory[] hashFunctionFactories = JobGenHelper.variablesToBinaryHashFunctionFactories(
gbyCols, aggOpInputEnv, context);
- ISerializableAggregateFunctionFactory[] merges = new ISerializableAggregateFunctionFactory[n];
+ ICopySerializableAggregateFunctionFactory[] merges = new ICopySerializableAggregateFunctionFactory[n];
List<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
IOperatorSchema[] localInputSchemas = new IOperatorSchema[1];
localInputSchemas[0] = new OperatorSchemaImpl();
@@ -224,7 +225,7 @@
for (i = 0; i < n; i++) {
AggregateFunctionCallExpression mergeFun = (AggregateFunctionCallExpression) aggOp.getMergeExpressions()
.get(i).getValue();
- merges[i] = exprJobGen.createSerializableAggregateFunctionFactory(mergeFun, aggOpInputEnv,
+ merges[i] = expressionRuntimeProvider.createSerializableAggregateFunctionFactory(mergeFun, aggOpInputEnv,
localInputSchemas, context);
}
IAggregatorDescriptorFactory aggregatorFactory = new SerializableAggregatorDescriptorFactory(aff);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionExchangePOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionExchangePOperator.java
index 823ac0b..043cb81 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionExchangePOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionExchangePOperator.java
@@ -14,15 +14,16 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-import java.util.HashSet;
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryHashFunctionFactoryProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
@@ -31,13 +32,12 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
@@ -66,7 +66,7 @@
@Override
public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
- IPartitioningProperty p = new UnorderedPartitionedProperty(new HashSet<LogicalVariable>(hashFields), domain);
+ IPartitioningProperty p = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(hashFields), domain);
this.deliveredProperties = new StructuralPropertiesVector(p, null);
}
@@ -82,7 +82,7 @@
}
@Override
- public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(JobSpecification spec,
+ public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
int[] keys = new int[hashFields.size()];
IBinaryHashFunctionFactory[] hashFunctionFactories = new IBinaryHashFunctionFactory[hashFields.size()];
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java
index a54f116..f3c9e5a 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java
@@ -15,17 +15,17 @@
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
import java.util.ArrayList;
-import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryHashFunctionFactoryProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
@@ -41,14 +41,14 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningMergingConnectorDescriptor;
@@ -76,7 +76,7 @@
@Override
public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
- IPartitioningProperty p = new UnorderedPartitionedProperty(new HashSet<LogicalVariable>(partitionFields),
+ IPartitioningProperty p = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(partitionFields),
domain);
AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
List<ILocalStructuralProperty> op2Locals = op2.getDeliveredPhysicalProperties().getLocalProperties();
@@ -123,7 +123,7 @@
}
@Override
- public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(JobSpecification spec,
+ public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
int[] keys = new int[partitionFields.size()];
IBinaryHashFunctionFactory[] hashFunctionFactories = new IBinaryHashFunctionFactory[partitionFields.size()];
@@ -149,7 +149,7 @@
sortFields[j] = opSchema.findVariable(var);
Object type = env.getVarType(var);
IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
- comparatorFactories[j] = bcfp.getBinaryComparatorFactory(type, oc.getOrder());
+ comparatorFactories[j] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
j++;
}
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java
index 2d3724a..c737cc4 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java
@@ -17,29 +17,28 @@
import java.util.LinkedList;
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.join.HybridHashJoinOperatorDescriptor;
public class HybridHashJoinPOperator extends AbstractHashJoinPOperator {
@@ -96,10 +95,10 @@
IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
for (LogicalVariable v : keysLeftBranch) {
Object t = env.getVarType(v);
- comparatorFactories[i++] = bcfp.getBinaryComparatorFactory(t, OrderKind.ASC);
+ comparatorFactories[i++] = bcfp.getBinaryComparatorFactory(t, true);
}
- RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
- JobSpecification spec = builder.getJobSpec();
+ RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
+ IOperatorDescriptorRegistry spec = builder.getJobSpec();
IOperatorDescriptor opDesc = null;
try {
switch (kind) {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java
index 08406d4..563fcc5 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java
@@ -17,12 +17,13 @@
import java.util.LinkedList;
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
@@ -30,16 +31,15 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.join.InMemoryHashJoinOperatorDescriptor;
public class InMemoryHashJoinPOperator extends AbstractHashJoinPOperator {
@@ -85,10 +85,10 @@
IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
for (LogicalVariable v : keysLeftBranch) {
Object t = env.getVarType(v);
- comparatorFactories[i++] = bcfp.getBinaryComparatorFactory(t, OrderKind.ASC);
+ comparatorFactories[i++] = bcfp.getBinaryComparatorFactory(t, true);
}
- RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
- JobSpecification spec = builder.getJobSpec();
+ RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
+ IOperatorDescriptorRegistry spec = builder.getJobSpec();
IOperatorDescriptor opDesc = null;
switch (kind) {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java
index 8b8138f..ec2862e 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java
@@ -14,21 +14,21 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.INormalizedKeyComputerFactoryProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.sort.InMemorySortRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.sort.InMemorySortRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
@@ -52,7 +52,7 @@
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
throws AlgebricksException {
- RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
int n = sortColumns.length;
int[] sortFields = new int[n];
IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
@@ -66,11 +66,11 @@
Object type = env.getVarType(var);
OrderKind order = oc.getOrder();
if (i == 0 && nkcfProvider != null && type != null) {
- nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order);
+ nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order == OrderKind.ASC);
}
IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
- comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder());
+ comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
i++;
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
index 276122f..e000b85 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
@@ -4,11 +4,18 @@
import java.util.LinkedList;
import java.util.List;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
@@ -24,25 +31,28 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.job.JobSpecification;
public class IndexInsertDeletePOperator extends AbstractPhysicalOperator {
- private List<LogicalVariable> primaryKeys;
- private List<LogicalVariable> secondaryKeys;
- private IDataSourceIndex<?, ?> dataSourceIndex;
+ private final List<LogicalVariable> primaryKeys;
+ private final List<LogicalVariable> secondaryKeys;
+ private final ILogicalExpression filterExpr;
+ private final IDataSourceIndex<?, ?> dataSourceIndex;
public IndexInsertDeletePOperator(List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
- IDataSourceIndex<?, ?> dataSourceIndex) {
+ Mutable<ILogicalExpression> filterExpr, IDataSourceIndex<?, ?> dataSourceIndex) {
this.primaryKeys = primaryKeys;
this.secondaryKeys = secondaryKeys;
+ if (filterExpr != null) {
+ this.filterExpr = filterExpr.getValue();
+ } else {
+ this.filterExpr = null;
+ }
this.dataSourceIndex = dataSourceIndex;
}
@@ -54,7 +64,7 @@
@Override
public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
- deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+ deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
}
@Override
@@ -83,17 +93,18 @@
IMetadataProvider mp = context.getMetadataProvider();
JobSpecification spec = builder.getJobSpec();
- RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getValue(), inputSchemas[0],
+ RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0],
context);
Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = null;
- if (insertDeleteOp.getOperation() == Kind.INSERT)
- runtimeAndConstraints = mp.getIndexInsertRuntime(dataSourceIndex, propagatedSchema, primaryKeys,
- secondaryKeys, inputDesc, context, spec);
- else
- runtimeAndConstraints = mp.getIndexDeleteRuntime(dataSourceIndex, propagatedSchema, primaryKeys,
- secondaryKeys, inputDesc, context, spec);
-
+ IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(insertDeleteOp);
+ if (insertDeleteOp.getOperation() == Kind.INSERT) {
+ runtimeAndConstraints = mp.getIndexInsertRuntime(dataSourceIndex, propagatedSchema, inputSchemas, typeEnv,
+ primaryKeys, secondaryKeys, filterExpr, inputDesc, context, spec);
+ } else {
+ runtimeAndConstraints = mp.getIndexDeleteRuntime(dataSourceIndex, propagatedSchema, inputSchemas, typeEnv,
+ primaryKeys, secondaryKeys, filterExpr, inputDesc, context, spec);
+ }
builder.contributeHyracksOperator(insertDeleteOp, runtimeAndConstraints.first);
builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
ILogicalOperator src = insertDeleteOp.getInputs().get(0).getValue();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
index 2863aa6..477d257 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
@@ -3,6 +3,9 @@
import java.util.ArrayList;
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -18,11 +21,8 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.job.JobSpecification;
@@ -71,7 +71,7 @@
InsertDeleteOperator insertDeleteOp = (InsertDeleteOperator) op;
IMetadataProvider mp = context.getMetadataProvider();
JobSpecification spec = builder.getJobSpec();
- RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getValue(), inputSchemas[0],
+ RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0],
context);
Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = null;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java
index 5b02d83..c159ffd 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java
@@ -4,6 +4,8 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
@@ -14,13 +16,11 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg.NestedPlansAccumulatingAggregatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.group.MicroPreClusteredGroupRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.NestedPlansAccumulatingAggregatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.group.MicroPreClusteredGroupRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
@@ -67,8 +67,8 @@
IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(
columnList, env, context);
- RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
- RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getValue(),
+ RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
+ RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()),
inputSchemas[0], context);
MicroPreClusteredGroupRuntimeFactory runtime = new MicroPreClusteredGroupRuntimeFactory(keys,
comparatorFactories, aggregatorFactory, inputRecordDesc, recordDescriptor, null);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java
index 5978b10..8cbd2d8 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java
@@ -17,12 +17,13 @@
import java.util.LinkedList;
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryBooleanInspector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
@@ -34,30 +35,29 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspector;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparator;
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
import edu.uci.ics.hyracks.dataflow.std.join.NestedLoopJoinOperatorDescriptor;
/**
- *
* Left input is broadcast and preserves its local properties.
- *
* Right input can be partitioned in any way.
- *
*/
public class NLJoinPOperator extends AbstractJoinPOperator {
@@ -97,7 +97,7 @@
pp = pv1.getPartitioningProperty();
}
} else {
- pp = null;
+ pp = IPartitioningProperty.UNPARTITIONED;
}
List<ILocalStructuralProperty> localProps = new LinkedList<ILocalStructuralProperty>();
@@ -122,15 +122,15 @@
IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
throws AlgebricksException {
AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op;
- RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
IOperatorSchema[] conditionInputSchemas = new IOperatorSchema[1];
conditionInputSchemas[0] = propagatedSchema;
- ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
- IEvaluatorFactory cond = exprJobGen.createEvaluatorFactory(join.getCondition().getValue(),
+ IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
+ IScalarEvaluatorFactory cond = expressionRuntimeProvider.createEvaluatorFactory(join.getCondition().getValue(),
context.getTypeEnvironment(op), conditionInputSchemas, context);
ITuplePairComparatorFactory comparatorFactory = new TuplePairEvaluatorFactory(cond,
- context.getBinaryBooleanInspector());
- JobSpecification spec = builder.getJobSpec();
+ context.getBinaryBooleanInspectorFactory());
+ IOperatorDescriptorRegistry spec = builder.getJobSpec();
IOperatorDescriptor opDesc = null;
switch (kind) {
@@ -154,36 +154,38 @@
public static class TuplePairEvaluatorFactory implements ITuplePairComparatorFactory {
private static final long serialVersionUID = 1L;
- private final IEvaluatorFactory cond;
- private final IBinaryBooleanInspector binaryBooleanInspector;
+ private final IScalarEvaluatorFactory cond;
+ private final IBinaryBooleanInspectorFactory binaryBooleanInspectorFactory;
- public TuplePairEvaluatorFactory(IEvaluatorFactory cond, IBinaryBooleanInspector binaryBooleanInspector) {
+ public TuplePairEvaluatorFactory(IScalarEvaluatorFactory cond,
+ IBinaryBooleanInspectorFactory binaryBooleanInspectorFactory) {
this.cond = cond;
- this.binaryBooleanInspector = binaryBooleanInspector;
+ this.binaryBooleanInspectorFactory = binaryBooleanInspectorFactory;
}
@Override
- public synchronized ITuplePairComparator createTuplePairComparator() {
- return new TuplePairEvaluator(cond, binaryBooleanInspector);
+ public synchronized ITuplePairComparator createTuplePairComparator(IHyracksTaskContext ctx) {
+ return new TuplePairEvaluator(ctx, cond, binaryBooleanInspectorFactory.createBinaryBooleanInspector(ctx));
}
-
}
public static class TuplePairEvaluator implements ITuplePairComparator {
-
- private IEvaluator condEvaluator;
- private final IEvaluatorFactory condFactory;
+ private final IHyracksTaskContext ctx;
+ private IScalarEvaluator condEvaluator;
+ private final IScalarEvaluatorFactory condFactory;
+ private final IPointable p;
private final CompositeFrameTupleReference compositeTupleRef;
private final FrameTupleReference leftRef;
private final FrameTupleReference rightRef;
- private final ArrayBackedValueStorage evalOutput;
private final IBinaryBooleanInspector binaryBooleanInspector;
- public TuplePairEvaluator(IEvaluatorFactory condFactory, IBinaryBooleanInspector binaryBooleanInspector) {
+ public TuplePairEvaluator(IHyracksTaskContext ctx, IScalarEvaluatorFactory condFactory,
+ IBinaryBooleanInspector binaryBooleanInspector) {
+ this.ctx = ctx;
this.condFactory = condFactory;
this.binaryBooleanInspector = binaryBooleanInspector;
- this.evalOutput = new ArrayBackedValueStorage();
this.leftRef = new FrameTupleReference();
+ this.p = VoidPointable.FACTORY.createPointable();
this.rightRef = new FrameTupleReference();
this.compositeTupleRef = new CompositeFrameTupleReference(leftRef, rightRef);
}
@@ -193,19 +195,19 @@
int innerIndex) throws HyracksDataException {
if (condEvaluator == null) {
try {
- this.condEvaluator = condFactory.createEvaluator(evalOutput);
+ this.condEvaluator = condFactory.createScalarEvaluator(ctx);
} catch (AlgebricksException ae) {
throw new HyracksDataException(ae);
}
}
compositeTupleRef.reset(outerAccessor, outerIndex, innerAccessor, innerIndex);
try {
- condEvaluator.evaluate(compositeTupleRef);
+ condEvaluator.evaluate(compositeTupleRef, p);
} catch (AlgebricksException ae) {
throw new HyracksDataException(ae);
}
- boolean result = binaryBooleanInspector.getBooleanValue(evalOutput.getBytes(), 0, evalOutput.getLength());
- evalOutput.reset();
+ boolean result = binaryBooleanInspector
+ .getBooleanValue(p.getByteArray(), p.getStartOffset(), p.getLength());
if (result)
return 0;
else
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java
index 5a4a204..47a3dd2 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java
@@ -16,6 +16,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -26,10 +27,9 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.NestedTupleSourceRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.NestedTupleSourceRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
public class NestedTupleSourcePOperator extends AbstractPhysicalOperator {
@@ -67,7 +67,7 @@
throws AlgebricksException {
propagatedSchema.addAllVariables(outerPlanSchema);
NestedTupleSourceRuntimeFactory runtime = new NestedTupleSourceRuntimeFactory();
- RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
builder.contributeMicroOperator(op, runtime, recDesc);
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
index 0ed17d8..eb4fffa 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
@@ -14,6 +14,7 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -23,10 +24,9 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
public class OneToOneExchangePOperator extends AbstractExchangePOperator {
@@ -52,7 +52,7 @@
}
@Override
- public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(JobSpecification spec,
+ public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) {
IConnectorDescriptor conn = new OneToOneConnectorDescriptor(spec);
return new Pair<IConnectorDescriptor, TargetConstraint>(conn, TargetConstraint.SAME_COUNT);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java
index d74fb62..29af97c 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java
@@ -15,9 +15,10 @@
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
import java.util.ArrayList;
-import java.util.HashSet;
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -36,16 +37,15 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg.SimpleAlgebricksAccumulatingAggregatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.SimpleAlgebricksAccumulatingAggregatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.preclustered.PreclusteredGroupOperatorDescriptor;
public class PreSortedDistinctByPOperator extends AbstractPhysicalOperator {
@@ -83,7 +83,7 @@
IPartitioningProperty pp = null;
AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
- pp = new UnorderedPartitionedProperty(new HashSet<LogicalVariable>(columnList), null);
+ pp = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(columnList), null);
}
pv[0] = new StructuralPropertiesVector(pp, localProps);
return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
@@ -94,7 +94,7 @@
IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
throws AlgebricksException {
- JobSpecification spec = builder.getJobSpec();
+ IOperatorDescriptorRegistry spec = builder.getJobSpec();
int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchemas[0]);
int sz = inputSchemas[0].getSize();
int fdSz = sz - columnList.size();
@@ -107,11 +107,11 @@
}
IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(
columnList, context.getTypeEnvironment(op), context);
- IAggregateFunctionFactory[] aggFactories = new IAggregateFunctionFactory[] {};
+ IAggregateEvaluatorFactory[] aggFactories = new IAggregateEvaluatorFactory[] {};
IAggregatorDescriptorFactory aggregatorFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(
aggFactories, keys, fdColumns);
- RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
PreclusteredGroupOperatorDescriptor opDesc = new PreclusteredGroupOperatorDescriptor(spec, keys,
comparatorFactories, aggregatorFactory, recordDescriptor);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java
index bb19bb9..0849c7d 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java
@@ -18,6 +18,8 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
@@ -28,17 +30,15 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg.NestedPlansAccumulatingAggregatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.NestedPlansAccumulatingAggregatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.preclustered.PreclusteredGroupOperatorDescriptor;
public class PreclusteredGroupByPOperator extends AbstractPreclusteredGroupByPOperator {
@@ -79,10 +79,10 @@
IAggregatorDescriptorFactory aggregatorFactory = new NestedPlansAccumulatingAggregatorFactory(subplans, keys,
fdColumns);
- JobSpecification spec = builder.getJobSpec();
+ IOperatorDescriptorRegistry spec = builder.getJobSpec();
IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(
columnList, context.getTypeEnvironment(op), context);
- RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
PreclusteredGroupOperatorDescriptor opDesc = new PreclusteredGroupOperatorDescriptor(spec, keys,
comparatorFactories, aggregatorFactory, recordDescriptor);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java
index abfa31f..22d8cc3 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java
@@ -16,20 +16,20 @@
import java.util.ArrayList;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.connectors.MToNReplicatingConnectorDescriptor;
public class RandomMergeExchangePOperator extends AbstractExchangePOperator {
@@ -52,7 +52,7 @@
}
@Override
- public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(JobSpecification spec,
+ public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) {
IConnectorDescriptor conn = new MToNReplicatingConnectorDescriptor(spec);
return new Pair<IConnectorDescriptor, TargetConstraint>(conn, TargetConstraint.ONE);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RangePartitionPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RangePartitionPOperator.java
index ff558ec..7e3c935 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RangePartitionPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RangePartitionPOperator.java
@@ -17,10 +17,13 @@
import java.util.ArrayList;
import java.util.LinkedList;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
@@ -30,12 +33,9 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderedPartitionedProperty;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
public class RangePartitionPOperator extends AbstractExchangePOperator {
@@ -65,7 +65,7 @@
}
@Override
- public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(JobSpecification spec,
+ public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
throw new NotImplementedException();
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java
index b42dd5b..060e8d7 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java
@@ -14,6 +14,7 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -24,11 +25,10 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.misc.SplitOperatorDescriptor;
public class ReplicatePOperator extends AbstractPhysicalOperator {
@@ -59,8 +59,8 @@
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
throws AlgebricksException {
- JobSpecification spec = builder.getJobSpec();
- RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ IOperatorDescriptorRegistry spec = builder.getJobSpec();
+ RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
ReplicateOperator rop = (ReplicateOperator) op;
int outputArity = rop.getOutputArity();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java
index 3647deb..b37c0ea 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java
@@ -18,13 +18,14 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
@@ -34,11 +35,10 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IRunningAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.RunningAggregateRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.RunningAggregateRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
public class RunningAggregatePOperator extends AbstractPhysicalOperator {
@@ -91,11 +91,11 @@
for (int i = 0; i < outColumns.length; i++) {
outColumns[i] = opSchema.findVariable(variables.get(i));
}
- IRunningAggregateFunctionFactory[] runningAggFuns = new IRunningAggregateFunctionFactory[expressions.size()];
- ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
+ IRunningAggregateEvaluatorFactory[] runningAggFuns = new IRunningAggregateEvaluatorFactory[expressions.size()];
+ IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
for (int i = 0; i < runningAggFuns.length; i++) {
StatefulFunctionCallExpression expr = (StatefulFunctionCallExpression) expressions.get(i).getValue();
- runningAggFuns[i] = exprJobGen.createRunningAggregateFunctionFactory(expr,
+ runningAggFuns[i] = expressionRuntimeProvider.createRunningAggregateFunctionFactory(expr,
context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
}
@@ -106,7 +106,7 @@
projectionList);
// contribute one Asterix framewriter
- RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
builder.contributeMicroOperator(ragg, runtime, recDesc);
// and contribute one edge from its child
ILogicalOperator src = ragg.getInputs().get(0).getValue();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java
index 0375246..ccb61a2 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java
@@ -14,6 +14,7 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -23,10 +24,9 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.SinkRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.SinkRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
public class SinkPOperator extends AbstractPhysicalOperator {
@@ -57,7 +57,7 @@
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
throws AlgebricksException {
- RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
SinkRuntimeFactory runtime = new SinkRuntimeFactory();
builder.contributeMicroOperator(op, runtime, recDesc);
ILogicalOperator src = op.getInputs().get(0).getValue();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java
index 123b0e1..e89b8e0 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java
@@ -16,6 +16,10 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
@@ -23,7 +27,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
@@ -34,13 +37,10 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
public class SinkWritePOperator extends AbstractPhysicalOperator {
@@ -87,8 +87,8 @@
LogicalVariable v = varRef.getVariableReference();
columns[i++] = inputSchemas[0].findVariable(v);
}
- RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
- RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getValue(), inputSchemas[0],
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
+ RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0],
context);
IPrinterFactory[] pf = JobGenHelper.mkPrinterFactories(inputSchemas[0], context.getTypeEnvironment(op),
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java
index 98c9be3..19d6c79 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java
@@ -17,15 +17,16 @@
import java.util.ArrayList;
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryHashFunctionFactoryProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
@@ -35,14 +36,14 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningMergingConnectorDescriptor;
@@ -120,7 +121,7 @@
}
@Override
- public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(JobSpecification spec,
+ public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
int n = sortColumns.length;
int[] sortFields = new int[n];
@@ -131,7 +132,7 @@
sortFields[i] = opSchema.findVariable(sortColumns[i].getColumn());
Object type = env.getVarType(sortColumns[i].getColumn());
IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
- comps[i] = bcfp.getBinaryComparatorFactory(type, sortColumns[i].getOrder());
+ comps[i] = bcfp.getBinaryComparatorFactory(type, sortColumns[i].getOrder() == OrderKind.ASC);
IBinaryHashFunctionFactoryProvider bhffp = context.getBinaryHashFunctionFactoryProvider();
hashFuns[i] = bhffp.getBinaryHashFunctionFactory(type);
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java
index 73c7dd6..df15f29 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java
@@ -14,32 +14,29 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.INormalizedKeyComputerFactoryProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
/**
- *
* This will always be attached to an {@link OrderOperator} logical operator.
- *
- *
*/
public class StableSortPOperator extends AbstractStableSortPOperator {
@@ -65,8 +62,8 @@
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
throws AlgebricksException {
- JobSpecification spec = builder.getJobSpec();
- RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ IOperatorDescriptorRegistry spec = builder.getJobSpec();
+ RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
int n = sortColumns.length;
int[] sortFields = new int[n];
IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
@@ -82,10 +79,10 @@
Object type = env.getVarType(var);
OrderKind order = oc.getOrder();
if (i == 0 && nkcfProvider != null && type != null) {
- nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order);
+ nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order == OrderKind.ASC);
}
IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
- comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder());
+ comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
i++;
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamDiePOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamDiePOperator.java
index 82ea31c..73b9b79 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamDiePOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamDiePOperator.java
@@ -14,11 +14,12 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DieOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
@@ -26,11 +27,10 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StreamDieRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StreamDieRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
public class StreamDiePOperator extends AbstractPhysicalOperator {
@@ -67,13 +67,13 @@
IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
throws AlgebricksException {
DieOperator die = (DieOperator) op;
- ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
+ IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
IVariableTypeEnvironment env = context.getTypeEnvironment(op);
- IEvaluatorFactory afterObjectsFact = exprJobGen.createEvaluatorFactory(die.getAfterObjects().getValue(),
- env, inputSchemas, context);
- RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ IScalarEvaluatorFactory afterObjectsFact = expressionRuntimeProvider.createEvaluatorFactory(die
+ .getAfterObjects().getValue(), env, inputSchemas, context);
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
StreamDieRuntimeFactory runtime = new StreamDieRuntimeFactory(afterObjectsFact, null,
- context.getBinaryIntegerInspector());
+ context.getBinaryIntegerInspectorFactory());
builder.contributeMicroOperator(die, runtime, recDesc);
ILogicalOperator src = die.getInputs().get(0).getValue();
builder.contributeGraphEdge(src, 0, die, 0);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java
index a31b2da..11e24d7 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java
@@ -14,12 +14,13 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
@@ -28,11 +29,10 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StreamLimitRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StreamLimitRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
public class StreamLimitPOperator extends AbstractPhysicalOperator {
@@ -76,16 +76,16 @@
IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
throws AlgebricksException {
LimitOperator limit = (LimitOperator) op;
- ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
+ IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
IVariableTypeEnvironment env = context.getTypeEnvironment(op);
- IEvaluatorFactory maxObjectsFact = exprJobGen.createEvaluatorFactory(limit.getMaxObjects().getValue(),
- env, inputSchemas, context);
+ IScalarEvaluatorFactory maxObjectsFact = expressionRuntimeProvider.createEvaluatorFactory(limit.getMaxObjects()
+ .getValue(), env, inputSchemas, context);
ILogicalExpression offsetExpr = limit.getOffset().getValue();
- IEvaluatorFactory offsetFact = (offsetExpr == null) ? null : exprJobGen.createEvaluatorFactory(offsetExpr, env,
- inputSchemas, context);
- RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ IScalarEvaluatorFactory offsetFact = (offsetExpr == null) ? null : expressionRuntimeProvider
+ .createEvaluatorFactory(offsetExpr, env, inputSchemas, context);
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
StreamLimitRuntimeFactory runtime = new StreamLimitRuntimeFactory(maxObjectsFact, offsetFact, null,
- context.getBinaryIntegerInspector());
+ context.getBinaryIntegerInspectorFactory());
builder.contributeMicroOperator(limit, runtime, recDesc);
// and contribute one edge from its child
ILogicalOperator src = limit.getInputs().get(0).getValue();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java
index 94e2135..54c0505 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java
@@ -14,6 +14,7 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -23,10 +24,9 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StreamProjectRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StreamProjectRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
public class StreamProjectPOperator extends AbstractPropagatePropertiesForUsedVariablesPOperator {
@@ -62,7 +62,7 @@
projectionList[i++] = pos;
}
StreamProjectRuntimeFactory runtime = new StreamProjectRuntimeFactory(projectionList);
- RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
builder.contributeMicroOperator(project, runtime, recDesc);
ILogicalOperator src = project.getInputs().get(0).getValue();
builder.contributeGraphEdge(src, 0, project, 0);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java
index b4112c8..4352d1f 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java
@@ -14,20 +14,20 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StreamSelectRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StreamSelectRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
public class StreamSelectPOperator extends AbstractPhysicalOperator {
@@ -59,13 +59,13 @@
IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
throws AlgebricksException {
SelectOperator select = (SelectOperator) op;
- ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
- IEvaluatorFactory cond = exprJobGen.createEvaluatorFactory(select.getCondition().getValue(),
- context.getTypeEnvironment(op), inputSchemas, context);
+ IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
+ IScalarEvaluatorFactory cond = expressionRuntimeProvider.createEvaluatorFactory(select.getCondition()
+ .getValue(), context.getTypeEnvironment(op), inputSchemas, context);
StreamSelectRuntimeFactory runtime = new StreamSelectRuntimeFactory(cond, null,
- context.getBinaryBooleanInspector());
+ context.getBinaryBooleanInspectorFactory());
// contribute one Asterix framewriter
- RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
builder.contributeMicroOperator(select, runtime, recDesc);
// and contribute one edge from its child
ILogicalOperator src = select.getInputs().get(0).getValue();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java
index dfab5ed..52f8e0b 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java
@@ -14,6 +14,7 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -22,13 +23,12 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StringStreamingRuntimeFactory;
import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.IScriptDescription;
import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.IScriptDescription.ScriptKind;
import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.StringStreamingScriptDescription;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StringStreamingRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
public class StringStreamingScriptPOperator extends AbstractPropagatePropertiesForUsedVariablesPOperator {
@@ -61,7 +61,7 @@
StringStreamingScriptDescription sssd = (StringStreamingScriptDescription) scriptDesc;
StringStreamingRuntimeFactory runtime = new StringStreamingRuntimeFactory(sssd.getCommand(),
sssd.getPrinterFactories(), sssd.getFieldDelimiter(), sssd.getParserFactory());
- RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
builder.contributeMicroOperator(scriptOp, runtime, recDesc);
// and contribute one edge from its child
ILogicalOperator src = scriptOp.getInputs().get(0).getValue();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java
index 4e6ef69..933acce 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java
@@ -19,6 +19,8 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -31,12 +33,10 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta.SubplanRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.meta.SubplanRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
@@ -89,7 +89,7 @@
AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], subplan, opSchema, context);
assert (subplans.length == 1);
AlgebricksPipeline np = subplans[0];
- RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getValue(),
+ RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()),
inputSchemas[0], context);
INullWriterFactory[] nullWriterFactories = new INullWriterFactory[np.getOutputWidth()];
for (int i = 0; i < nullWriterFactories.length; i++) {
@@ -97,7 +97,7 @@
}
SubplanRuntimeFactory runtime = new SubplanRuntimeFactory(np, nullWriterFactories, inputRecordDesc, null);
- RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
builder.contributeMicroOperator(subplan, runtime, recDesc);
ILogicalOperator src = op.getInputs().get(0).getValue();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java
index da88dd9..d3f7851 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java
@@ -16,6 +16,8 @@
import java.util.ArrayList;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -30,12 +32,10 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.union.UnionAllOperatorDescriptor;
public class UnionAllPOperator extends AbstractPhysicalOperator {
@@ -84,8 +84,8 @@
++i;
}
- JobSpecification spec = builder.getJobSpec();
- RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ IOperatorDescriptorRegistry spec = builder.getJobSpec();
+ RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
// at algebricks level, union all only accepts two inputs, although at
// hyracks
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java
index 8f838ba..38ebec1 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java
@@ -14,6 +14,7 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
@@ -22,16 +23,15 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IUnnestingFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.UnnestRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.UnnestRuntimeFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
public class UnnestPOperator extends AbstractScanPOperator {
@@ -63,7 +63,7 @@
}
int outCol = opSchema.findVariable(unnest.getVariable());
ILogicalExpression unnestExpr = unnest.getExpressionRef().getValue();
- ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
+ IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
boolean exit = false;
if (unnestExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
exit = true;
@@ -77,11 +77,11 @@
throw new AlgebricksException("Unnest expression " + unnestExpr + " is not an unnesting function call.");
}
UnnestingFunctionCallExpression agg = (UnnestingFunctionCallExpression) unnestExpr;
- IUnnestingFunctionFactory unnestingFactory = exprJobGen.createUnnestingFunctionFactory(agg,
+ IUnnestingEvaluatorFactory unnestingFactory = expressionRuntimeProvider.createUnnestingFunctionFactory(agg,
context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
int[] projectionList = JobGenHelper.projectAllVariables(opSchema);
UnnestRuntimeFactory unnestRuntime = new UnnestRuntimeFactory(outCol, unnestingFactory, projectionList);
- RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
builder.contributeMicroOperator(unnest, unnestRuntime, recDesc);
ILogicalOperator src = unnest.getInputs().get(0).getValue();
builder.contributeGraphEdge(src, 0, unnest, 0);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
index d1f0340..547fe843 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
@@ -3,6 +3,9 @@
import java.util.ArrayList;
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -17,10 +20,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.job.JobSpecification;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
index 05ae1ec..a94c78e 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
@@ -18,6 +18,9 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -45,6 +48,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -52,9 +56,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
public class LogicalOperatorPrettyPrintVisitor implements ILogicalOperatorVisitor<String, Integer> {
@@ -223,8 +224,7 @@
@Override
public String visitUnnestMapOperator(UnnestMapOperator op, Integer indent) {
StringBuilder buffer = new StringBuilder();
- addIndent(buffer, indent).append(
- "unnest-map " + op.getVariables() + " <- " + op.getExpressionRef().getValue());
+ addIndent(buffer, indent).append("unnest-map " + op.getVariables() + " <- " + op.getExpressionRef().getValue());
return buffer.toString();
}
@@ -346,4 +346,11 @@
return buffer.toString();
}
+ @Override
+ public String visitExtensionOperator(ExtensionOperator op, Integer indent) throws AlgebricksException {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("statistics collection");
+ return buffer.toString();
+ }
+
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java
index 174db0b..aa96513 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java
@@ -16,12 +16,12 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class PlanPrettyPrinter {
public static void printPlan(ILogicalPlan plan, StringBuilder out, LogicalOperatorPrettyPrintVisitor pvisitor,
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/AbstractGroupingProperty.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/AbstractGroupingProperty.java
index c11131a..341b4f0 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/AbstractGroupingProperty.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/AbstractGroupingProperty.java
@@ -14,11 +14,11 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -43,7 +43,7 @@
if (equivalenceClasses == null || equivalenceClasses.isEmpty()) {
return;
}
- HashSet<LogicalVariable> norm = new HashSet<LogicalVariable>();
+ Set<LogicalVariable> norm = new ListSet<LogicalVariable>();
for (LogicalVariable v : columnSet) {
EquivalenceClass ec = equivalenceClasses.get(v);
if (ec == null) {
@@ -65,7 +65,7 @@
if (fds == null || fds.isEmpty()) {
return;
}
- HashSet<LogicalVariable> norm = new HashSet<LogicalVariable>();
+ Set<LogicalVariable> norm = new ListSet<LogicalVariable>();
for (LogicalVariable v : columnSet) {
boolean isImpliedByAnFD = false;
for (FunctionalDependency fdep : fds) {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningRequirementsCoordinator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningRequirementsCoordinator.java
index d884a17..209f1e8 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningRequirementsCoordinator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningRequirementsCoordinator.java
@@ -14,23 +14,21 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
/**
- *
* Implements constraints in between requirements for the children of the same
* operator.
- *
*/
public interface IPartitioningRequirementsCoordinator {
@@ -57,13 +55,16 @@
UnorderedPartitionedProperty upp1 = (UnorderedPartitionedProperty) firstDeliveredPartitioning;
Set<LogicalVariable> set1 = upp1.getColumnSet();
UnorderedPartitionedProperty uppreq = (UnorderedPartitionedProperty) rqdpp;
- Set<LogicalVariable> modifuppreq = new HashSet<LogicalVariable>();
+ Set<LogicalVariable> modifuppreq = new ListSet<LogicalVariable>();
Map<LogicalVariable, EquivalenceClass> eqmap = context.getEquivalenceClassMap(op);
- Set<LogicalVariable> covered = new HashSet<LogicalVariable>();
- for (LogicalVariable r : uppreq.getColumnSet()) {
- EquivalenceClass ec = eqmap.get(r);
- for (LogicalVariable v : set1) {
- EquivalenceClass ecFirst = eqmap.get(v);
+ Set<LogicalVariable> covered = new ListSet<LogicalVariable>();
+
+ // coordinate from an existing partition property
+ // (firstDeliveredPartitioning)
+ for (LogicalVariable v : set1) {
+ EquivalenceClass ecFirst = eqmap.get(v);
+ for (LogicalVariable r : uppreq.getColumnSet()) {
+ EquivalenceClass ec = eqmap.get(r);
if (ecFirst == ec) {
covered.add(v);
modifuppreq.add(r);
@@ -71,6 +72,7 @@
}
}
}
+
if (!covered.equals(set1)) {
throw new AlgebricksException("Could not modify " + rqdpp
+ " to agree with partitioning property " + firstDeliveredPartitioning
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PropertiesUtil.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PropertiesUtil.java
index ad80e46..8ca4ccf 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PropertiesUtil.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PropertiesUtil.java
@@ -16,7 +16,6 @@
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -24,6 +23,7 @@
import java.util.Map;
import java.util.Set;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
@@ -31,7 +31,7 @@
public class PropertiesUtil {
public Set<LogicalVariable> closureUnderFDs(Collection<LogicalVariable> vars, List<FunctionalDependency> fdList) {
- Set<LogicalVariable> k = new HashSet<LogicalVariable>(vars);
+ Set<LogicalVariable> k = new ListSet<LogicalVariable>(vars);
boolean change;
do {
change = false;
@@ -65,8 +65,8 @@
ListIterator<ILocalStructuralProperty> dlvdIter = dlvd.listIterator();
- Set<LogicalVariable> rqdCols = new HashSet<LogicalVariable>();
- Set<LogicalVariable> dlvdCols = new HashSet<LogicalVariable>();
+ Set<LogicalVariable> rqdCols = new ListSet<LogicalVariable>();
+ Set<LogicalVariable> dlvdCols = new ListSet<LogicalVariable>();
for (ILocalStructuralProperty r : reqd) {
if (r.getPropertyType() == PropertyType.LOCAL_GROUPING_PROPERTY) {
rqdCols.clear();
@@ -180,7 +180,6 @@
}
/**
- *
* @param pref
* @param target
* @return true iff pref is a prefix of target
@@ -300,7 +299,7 @@
ILocalStructuralProperty p = propIter.previous();
ListIterator<ILocalStructuralProperty> secondIter = props.listIterator(pos);
pos--;
- Set<LogicalVariable> cols = new HashSet<LogicalVariable>();
+ Set<LogicalVariable> cols = new ListSet<LogicalVariable>();
while (secondIter.hasPrevious()) {
secondIter.previous().getColumns(cols);
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java
index 6b14a9f..01c2de4 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java
@@ -2,11 +2,11 @@
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public abstract class TypePropagationPolicy {
public static final TypePropagationPolicy ALL = new TypePropagationPolicy() {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/VariablePropagationPolicy.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/VariablePropagationPolicy.java
index 6f936c0..f9c4ac3 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/VariablePropagationPolicy.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/VariablePropagationPolicy.java
@@ -14,9 +14,9 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public abstract class VariablePropagationPolicy {
public static final VariablePropagationPolicy ALL = new VariablePropagationPolicy() {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/aggregators/TupleCountAggregateFunctionFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/aggregators/TupleCountAggregateFunctionFactory.java
deleted file mode 100644
index acb2e0e..0000000
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/aggregators/TupleCountAggregateFunctionFactory.java
+++ /dev/null
@@ -1,54 +0,0 @@
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.aggregators;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunction;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class TupleCountAggregateFunctionFactory implements IAggregateFunctionFactory {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public IAggregateFunction createAggregateFunction(IDataOutputProvider provider) throws AlgebricksException {
-
- final DataOutput out = provider.getDataOutput();
- return new IAggregateFunction() {
-
- int cnt;
-
- @Override
- public void step(IFrameTupleReference tuple) throws AlgebricksException {
- ++cnt;
- }
-
- @Override
- public void init() throws AlgebricksException {
- cnt = 0;
- }
-
- @Override
- public void finish() throws AlgebricksException {
- try {
- out.writeInt(cnt);
- } catch (IOException e) {
- throw new AlgebricksException(e);
- }
- }
-
- @Override
- public void finishPartial() throws AlgebricksException {
- try {
- out.writeInt(cnt);
- } catch (IOException e) {
- throw new AlgebricksException(e);
- }
- }
- };
- }
-
-}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/aggregators/TupleCountRunningAggregateFunctionFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/aggregators/TupleCountRunningAggregateFunctionFactory.java
deleted file mode 100644
index 02fe250..0000000
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/aggregators/TupleCountRunningAggregateFunctionFactory.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.aggregators;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IRunningAggregateFunction;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IRunningAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class TupleCountRunningAggregateFunctionFactory implements IRunningAggregateFunctionFactory {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public IRunningAggregateFunction createRunningAggregateFunction(IDataOutputProvider provider)
- throws AlgebricksException {
-
- final DataOutput out = provider.getDataOutput();
-
- return new IRunningAggregateFunction() {
-
- int cnt;
-
- @Override
- public void step(IFrameTupleReference tuple) throws AlgebricksException {
- ++cnt;
- try {
- out.writeInt(cnt);
- } catch (IOException e) {
- throw new AlgebricksException(e);
- }
- }
-
- @Override
- public void init() throws AlgebricksException {
- cnt = 0;
- }
- };
- }
-
-}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IAggregateFunctionFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IAggregateFunctionFactory.java
deleted file mode 100644
index e70c052..0000000
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IAggregateFunctionFactory.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
-
-public interface IAggregateFunctionFactory extends Serializable {
- public IAggregateFunction createAggregateFunction(IDataOutputProvider provider) throws AlgebricksException;
-}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IEvaluatorFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IEvaluatorFactory.java
deleted file mode 100644
index 46db040..0000000
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IEvaluatorFactory.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
-
-public interface IEvaluatorFactory extends Serializable {
- public IEvaluator createEvaluator(IDataOutputProvider output) throws AlgebricksException;
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/ISerializableAggregateFunctionFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/ISerializableAggregateFunctionFactory.java
deleted file mode 100644
index f9f871e..0000000
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/ISerializableAggregateFunctionFactory.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-
-public interface ISerializableAggregateFunctionFactory extends Serializable {
- public ISerializableAggregateFunction createAggregateFunction() throws AlgebricksException;
-}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunctionFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunctionFactory.java
deleted file mode 100644
index 2134fd6..0000000
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunctionFactory.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
-
-public interface IUnnestingFunctionFactory extends Serializable {
- public IUnnestingFunction createUnnestingFunction(IDataOutputProvider provider) throws AlgebricksException;
-}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/context/RuntimeContext.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/context/RuntimeContext.java
deleted file mode 100644
index 5afd17f..0000000
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/context/RuntimeContext.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-
-public class RuntimeContext {
- private IHyracksTaskContext hyracksContext;
-
- public RuntimeContext() {
- }
-
- public IHyracksTaskContext getHyracksContext() {
- return hyracksContext;
- }
-
- public void setHyracksContext(IHyracksTaskContext hyracksContext) {
- this.hyracksContext = hyracksContext;
- }
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/BinaryIntegerInspectorImpl.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/BinaryIntegerInspectorImpl.java
deleted file mode 100644
index 9d0587d..0000000
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/BinaryIntegerInspectorImpl.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryIntegerInspector;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-
-public class BinaryIntegerInspectorImpl implements IBinaryIntegerInspector {
-
- private static final long serialVersionUID = 1L;
- public static final BinaryIntegerInspectorImpl INSTANCE = new BinaryIntegerInspectorImpl();
-
- @Override
- public int getIntegerValue(byte[] bytes, int offset, int length) {
- return IntegerSerializerDeserializer.getInt(bytes, offset);
- }
-
-}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamDieRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamDieRuntimeFactory.java
deleted file mode 100644
index 86b9b3b..0000000
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamDieRuntimeFactory.java
+++ /dev/null
@@ -1,98 +0,0 @@
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
-
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryIntegerInspector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
-
-public class StreamDieRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
-
- private static final long serialVersionUID = 1L;
-
- private IEvaluatorFactory aftterObjectsEvalFactory;
- private IBinaryIntegerInspector binaryIntegerInspector;
-
- public StreamDieRuntimeFactory(IEvaluatorFactory maxObjectsEvalFactory, int[] projectionList,
- IBinaryIntegerInspector binaryIntegerInspector) {
- super(projectionList);
- this.aftterObjectsEvalFactory = maxObjectsEvalFactory;
- this.binaryIntegerInspector = binaryIntegerInspector;
- }
-
- @Override
- public String toString() {
- String s = "stream-die " + aftterObjectsEvalFactory.toString();
- return s;
- }
-
- @Override
- public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context) {
- return new AbstractOneInputOneOutputOneFramePushRuntime() {
-
- private IEvaluator evalAfterObjects;
- private ArrayBackedValueStorage evalOutput;
- private int toWrite = -1;
-
- @Override
- public void open() throws HyracksDataException {
- if (evalAfterObjects == null) {
- initAccessAppendRef(context);
- evalOutput = new ArrayBackedValueStorage();
- try {
- evalAfterObjects = aftterObjectsEvalFactory.createEvaluator(evalOutput);
- } catch (AlgebricksException ae) {
- throw new HyracksDataException(ae);
- }
- }
- writer.open();
- }
-
- @Override
- public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
- tAccess.reset(buffer);
- int nTuple = tAccess.getTupleCount();
- if (toWrite < 0) {
- toWrite = evaluateInteger(evalAfterObjects, 0);
- }
- for (int t = 0; t < nTuple; t++) {
- if (toWrite > 0) {
- toWrite--;
- if (projectionList != null) {
- appendProjectionToFrame(t, projectionList);
- } else {
- appendTupleToFrame(t);
- }
- } else {
- throw new HyracksDataException("injected failure");
- }
- }
- }
-
- @Override
- public void close() throws HyracksDataException {
- super.close();
- }
-
- private int evaluateInteger(IEvaluator eval, int tIdx) throws HyracksDataException {
- tRef.reset(tAccess, tIdx);
- evalOutput.reset();
- try {
- eval.evaluate(tRef);
- } catch (AlgebricksException ae) {
- throw new HyracksDataException(ae);
- }
- int lim = binaryIntegerInspector.getIntegerValue(evalOutput.getBytes(), 0, evalOutput.getLength());
- return lim;
- }
-
- };
- }
-
-}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamSelectRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamSelectRuntimeFactory.java
deleted file mode 100644
index be579e5..0000000
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamSelectRuntimeFactory.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
-
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryBooleanInspector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
-
-public class StreamSelectRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
-
- private static final long serialVersionUID = 1L;
-
- private IEvaluatorFactory cond;
-
- private IBinaryBooleanInspector binaryBooleanInspector;
-
- /**
- *
- * @param cond
- * @param projectionList
- * if projectionList is null, then no projection is performed
- */
- public StreamSelectRuntimeFactory(IEvaluatorFactory cond, int[] projectionList,
- IBinaryBooleanInspector binaryBooleanInspector) {
- super(projectionList);
- this.cond = cond;
- this.binaryBooleanInspector = binaryBooleanInspector;
- }
-
- @Override
- public String toString() {
- return "stream-select " + cond.toString();
- }
-
- @Override
- public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context) {
- return new AbstractOneInputOneOutputOneFramePushRuntime() {
-
- private IEvaluator eval;
- private ArrayBackedValueStorage evalOutput;
-
- @Override
- public void open() throws HyracksDataException {
- if (eval == null) {
- initAccessAppendRef(context);
- evalOutput = new ArrayBackedValueStorage();
- try {
- eval = cond.createEvaluator(evalOutput);
- } catch (AlgebricksException ae) {
- throw new HyracksDataException(ae);
- }
- }
- writer.open();
- }
-
- @Override
- public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
- tAccess.reset(buffer);
- int nTuple = tAccess.getTupleCount();
- for (int t = 0; t < nTuple; t++) {
- tRef.reset(tAccess, t);
- evalOutput.reset();
- try {
- eval.evaluate(tRef);
- } catch (AlgebricksException ae) {
- throw new HyracksDataException(ae);
- }
- if (binaryBooleanInspector.getBooleanValue(evalOutput.getBytes(), 0, evalOutput.getLength())) {
- if (projectionList != null) {
- appendProjectionToFrame(t, projectionList);
- } else {
- appendTupleToFrame(t);
- }
- }
- }
- }
-
- };
- }
-
-}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/IScriptDescription.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/IScriptDescription.java
index f292dd7..89f40d6 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/IScriptDescription.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/IScriptDescription.java
@@ -16,8 +16,8 @@
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
public interface IScriptDescription {
public enum ScriptKind {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/StringStreamingScriptDescription.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/StringStreamingScriptDescription.java
index 64cea03..f41535f 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/StringStreamingScriptDescription.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/StringStreamingScriptDescription.java
@@ -16,9 +16,9 @@
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
import edu.uci.ics.hyracks.dataflow.std.file.ITupleParserFactory;
public class StringStreamingScriptDescription implements IScriptDescription {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/AbstractTypeEnvironment.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/AbstractTypeEnvironment.java
index d1ebbab..e0217a9 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/AbstractTypeEnvironment.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/AbstractTypeEnvironment.java
@@ -3,12 +3,12 @@
import java.util.HashMap;
import java.util.Map;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public abstract class AbstractTypeEnvironment implements IVariableTypeEnvironment {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java
index e0f4e16..00cbac8 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java
@@ -2,10 +2,10 @@
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class NonPropagatingTypeEnvironment extends AbstractTypeEnvironment {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java
index 15504cb..5a42caf 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java
@@ -5,12 +5,12 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class PropagateOperatorInputsTypeEnvironment extends AbstractTypeEnvironment {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java
index bb2db54..d73e89f 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java
@@ -3,12 +3,12 @@
import java.util.ArrayList;
import java.util.List;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class PropagatingTypeEnvironment extends AbstractTypeEnvironment {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
index 4604198..47a979c 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
@@ -19,6 +19,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -32,7 +33,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class OperatorManipulationUtil {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java
index 97bd8ca..cd8f042 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java
@@ -20,6 +20,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -34,7 +35,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class OperatorPropertiesUtil {
@@ -169,15 +169,15 @@
return false;
}
AbstractFunctionCallExpression f1 = (AbstractFunctionCallExpression) eu;
- if (f1.getFunctionIdentifier() != AlgebricksBuiltinFunctions.NOT) {
+ if (!f1.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.NOT)) {
return false;
}
ILogicalExpression a1 = f1.getArguments().get(0).getValue();
- if (a1.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ if (!a1.getExpressionTag().equals(LogicalExpressionTag.FUNCTION_CALL)) {
return false;
}
AbstractFunctionCallExpression f2 = (AbstractFunctionCallExpression) a1;
- if (f2.getFunctionIdentifier() != AlgebricksBuiltinFunctions.IS_NULL) {
+ if (!f2.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.IS_NULL)) {
return false;
}
return true;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/AbstractConstVarFunVisitor.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/AbstractConstVarFunVisitor.java
index d4c5530..fac5052 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/AbstractConstVarFunVisitor.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/AbstractConstVarFunVisitor.java
@@ -14,12 +14,12 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.visitors;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public abstract class AbstractConstVarFunVisitor<R, T> implements ILogicalExpressionVisitor<R, T> {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java
index 2ad942a..a5808b1 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java
@@ -16,8 +16,8 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public interface ILogicalExpressionReferenceTransform {
// returns true if any change is made to the expression
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionVisitor.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionVisitor.java
index a299574..d64b6ae 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionVisitor.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionVisitor.java
@@ -14,13 +14,13 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.visitors;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public interface ILogicalExpressionVisitor<R, T> {
public R visitConstantExpression(ConstantExpression expr, T arg) throws AlgebricksException;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
index bfc7acc..6b5949e 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
@@ -14,6 +14,7 @@
*/
package edu.uci.ics.hyracks.algebricks.core.algebra.visitors;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
@@ -36,13 +37,13 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public interface ILogicalOperatorVisitor<R, T> {
@@ -68,6 +69,8 @@
public R visitSelectOperator(SelectOperator op, T arg) throws AlgebricksException;
+ public R visitExtensionOperator(ExtensionOperator op, T arg) throws AlgebricksException;
+
public R visitProjectOperator(ProjectOperator op, T arg) throws AlgebricksException;
public R visitPartitioningSplitOperator(PartitioningSplitOperator op, T arg) throws AlgebricksException;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/AlgebricksConfig.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/AlgebricksConfig.java
index 397f51b..11ec043 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/AlgebricksConfig.java
@@ -18,7 +18,7 @@
public class AlgebricksConfig {
public static final boolean DEBUG = true;
- public static final String ALGEBRICKS_LOGGER_NAME = "edu.uci.ics.algebricks";
+ public static final String ALGEBRICKS_LOGGER_NAME = "edu.uci.ics.hyracks.algebricks";
public static final Logger ALGEBRICKS_LOGGER = Logger.getLogger(ALGEBRICKS_LOGGER_NAME);
public static final String HYRACKS_APP_NAME = "algebricks";
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/ConnectorPolicyAssignmentPolicy.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/ConnectorPolicyAssignmentPolicy.java
new file mode 100644
index 0000000..e786f8c
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/ConnectorPolicyAssignmentPolicy.java
@@ -0,0 +1,24 @@
+package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
+
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
+import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicyAssignmentPolicy;
+import edu.uci.ics.hyracks.api.dataflow.connectors.PipeliningConnectorPolicy;
+import edu.uci.ics.hyracks.api.dataflow.connectors.SendSideMaterializedPipeliningConnectorPolicy;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningMergingConnectorDescriptor;
+
+public class ConnectorPolicyAssignmentPolicy implements IConnectorPolicyAssignmentPolicy {
+ private static final long serialVersionUID = 1L;
+ private IConnectorPolicy senderSideMaterializePolicy = new SendSideMaterializedPipeliningConnectorPolicy();
+ private IConnectorPolicy pipeliningPolicy = new PipeliningConnectorPolicy();
+
+ @Override
+ public IConnectorPolicy getConnectorPolicyAssignment(IConnectorDescriptor c, int nProducers, int nConsumers,
+ int[] fanouts) {
+ if (c instanceof MToNPartitioningMergingConnectorDescriptor) {
+ return senderSideMaterializePolicy;
+ } else {
+ return pipeliningPolicy;
+ }
+ }
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobBuilder.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
similarity index 95%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobBuilder.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
index 33102cb..61625d3 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobBuilder.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
@@ -12,22 +12,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl;
+package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksCountPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraintHelper;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobGenContext.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
similarity index 78%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobGenContext.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
index 870ce91..22a1a81 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobGenContext.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
@@ -12,33 +12,33 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl;
+package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
import java.util.HashMap;
import java.util.Map;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryBooleanInspector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryHashFunctionFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryIntegerInspector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.INormalizedKeyComputerFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.ISerializerDeserializerProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.ITypeTraitProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IPartialAggregationTypeComputer;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspectorFactory;
+import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.ISerializerDeserializerProvider;
+import edu.uci.ics.hyracks.algebricks.data.ITypeTraitProvider;
import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
public class JobGenContext {
@@ -53,9 +53,9 @@
private final INullWriterFactory nullWriterFactory;
private final INormalizedKeyComputerFactoryProvider normalizedKeyComputerFactoryProvider;
private final Object appContext;
- private final IBinaryBooleanInspector booleanInspector;
- private final IBinaryIntegerInspector integerInspector;
- private final ILogicalExpressionJobGen exprJobGen;
+ private final IBinaryBooleanInspectorFactory booleanInspectorFactory;
+ private final IBinaryIntegerInspectorFactory integerInspectorFactory;
+ private final IExpressionRuntimeProvider expressionRuntimeProvider;
private final IExpressionTypeComputer expressionTypeComputer;
private final IExpressionEvalSizeComputer expressionEvalSizeComputer;
private final IPartialAggregationTypeComputer partialAggregationTypeComputer;
@@ -68,10 +68,11 @@
ISerializerDeserializerProvider serializerDeserializerProvider,
IBinaryHashFunctionFactoryProvider hashFunctionFactoryProvider,
IBinaryComparatorFactoryProvider comparatorFactoryProvider, ITypeTraitProvider typeTraitProvider,
- IBinaryBooleanInspector booleanInspector, IBinaryIntegerInspector integerInspector,
- IPrinterFactoryProvider printerFactoryProvider, INullWriterFactory nullWriterFactory,
+ IBinaryBooleanInspectorFactory booleanInspectorFactory,
+ IBinaryIntegerInspectorFactory integerInspectorFactory, IPrinterFactoryProvider printerFactoryProvider,
+ INullWriterFactory nullWriterFactory,
INormalizedKeyComputerFactoryProvider normalizedKeyComputerFactoryProvider,
- ILogicalExpressionJobGen exprJobGen, IExpressionTypeComputer expressionTypeComputer,
+ IExpressionRuntimeProvider expressionRuntimeProvider, IExpressionTypeComputer expressionTypeComputer,
INullableTypeComputer nullableTypeComputer, ITypingContext typingContext,
IExpressionEvalSizeComputer expressionEvalSizeComputer,
IPartialAggregationTypeComputer partialAggregationTypeComputer, int frameSize,
@@ -83,13 +84,13 @@
this.hashFunctionFactoryProvider = hashFunctionFactoryProvider;
this.comparatorFactoryProvider = comparatorFactoryProvider;
this.typeTraitProvider = typeTraitProvider;
- this.booleanInspector = booleanInspector;
- this.integerInspector = integerInspector;
+ this.booleanInspectorFactory = booleanInspectorFactory;
+ this.integerInspectorFactory = integerInspectorFactory;
this.printerFactoryProvider = printerFactoryProvider;
this.clusterLocations = clusterLocations;
this.normalizedKeyComputerFactoryProvider = normalizedKeyComputerFactoryProvider;
this.nullWriterFactory = nullWriterFactory;
- this.exprJobGen = exprJobGen;
+ this.expressionRuntimeProvider = expressionRuntimeProvider;
this.expressionTypeComputer = expressionTypeComputer;
this.typingContext = typingContext;
this.expressionEvalSizeComputer = expressionEvalSizeComputer;
@@ -130,20 +131,20 @@
return typeTraitProvider;
}
- public IBinaryBooleanInspector getBinaryBooleanInspector() {
- return booleanInspector;
+ public IBinaryBooleanInspectorFactory getBinaryBooleanInspectorFactory() {
+ return booleanInspectorFactory;
}
- public IBinaryIntegerInspector getBinaryIntegerInspector() {
- return integerInspector;
+ public IBinaryIntegerInspectorFactory getBinaryIntegerInspectorFactory() {
+ return integerInspectorFactory;
}
public IPrinterFactoryProvider getPrinterFactoryProvider() {
return printerFactoryProvider;
}
- public ILogicalExpressionJobGen getExpressionJobGen() {
- return exprJobGen;
+ public IExpressionRuntimeProvider getExpressionRuntimeProvider() {
+ return expressionRuntimeProvider;
}
public IOperatorSchema getSchema(ILogicalOperator op) {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobGenHelper.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenHelper.java
similarity index 69%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobGenHelper.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenHelper.java
index 90bf47c..790fb93 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobGenHelper.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenHelper.java
@@ -12,24 +12,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl;
+package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
import java.util.Collection;
+import java.util.List;
import java.util.logging.Logger;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryHashFunctionFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.INormalizedKeyComputerFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.ISerializerDeserializerProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.ITypeTraitProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.ISerializerDeserializerProvider;
+import edu.uci.ics.hyracks.algebricks.data.ITypeTraitProvider;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
@@ -41,25 +40,26 @@
private static final Logger LOGGER = Logger.getLogger(JobGenHelper.class.getName());
- @SuppressWarnings("unchecked")
- public static RecordDescriptor mkRecordDescriptor(ILogicalOperator op, IOperatorSchema opSchema,
- JobGenContext context) throws AlgebricksException {
- ISerializerDeserializer[] fields = new ISerializerDeserializer[opSchema.getSize()];
+ @SuppressWarnings("rawtypes")
+ public static RecordDescriptor mkRecordDescriptor(IVariableTypeEnvironment env, IOperatorSchema opSchema,
+ JobGenContext context) throws AlgebricksException {
+ ISerializerDeserializer[] fields = new ISerializerDeserializer[opSchema.getSize()];
+ ITypeTraits[] typeTraits = new ITypeTraits[opSchema.getSize()];
ISerializerDeserializerProvider sdp = context.getSerializerDeserializerProvider();
+ ITypeTraitProvider ttp = context.getTypeTraitProvider();
int i = 0;
- IVariableTypeEnvironment env = context.getTypeEnvironment(op);
for (LogicalVariable var : opSchema) {
Object t = env.getVarType(var);
if (t == null) {
LOGGER.warning("No type for variable " + var);
- // throw new AlgebricksException("No type for variable " + var);
}
fields[i] = sdp.getSerializerDeserializer(t);
+ typeTraits[i] = ttp.getTypeTrait(t);
i++;
}
- return new RecordDescriptor(fields);
+ return new RecordDescriptor(fields, typeTraits);
}
-
+
public static IPrinterFactory[] mkPrinterFactories(IOperatorSchema opSchema, IVariableTypeEnvironment env,
JobGenContext context, int[] printColumns) throws AlgebricksException {
IPrinterFactory[] pf = new IPrinterFactory[printColumns.length];
@@ -94,7 +94,7 @@
}
return funFactories;
}
-
+
public static IBinaryComparatorFactory[] variablesToAscBinaryComparatorFactories(
Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context)
throws AlgebricksException {
@@ -103,7 +103,19 @@
int i = 0;
for (LogicalVariable v : varLogical) {
Object type = env.getVarType(v);
- compFactories[i++] = bcfProvider.getBinaryComparatorFactory(type, OrderKind.ASC);
+ compFactories[i++] = bcfProvider.getBinaryComparatorFactory(type, true);
+ }
+ return compFactories;
+ }
+
+ public static IBinaryComparatorFactory[] variablesToAscBinaryComparatorFactories(
+ List<LogicalVariable> varLogical, int start, int size, IVariableTypeEnvironment env, JobGenContext context)
+ throws AlgebricksException {
+ IBinaryComparatorFactory[] compFactories = new IBinaryComparatorFactory[size];
+ IBinaryComparatorFactoryProvider bcfProvider = context.getBinaryComparatorFactoryProvider();
+ for (int i = 0; i < size; i++) {
+ Object type = env.getVarType(varLogical.get(start + i));
+ compFactories[i] = bcfProvider.getBinaryComparatorFactory(type, true);
}
return compFactories;
}
@@ -116,7 +128,7 @@
return null;
for (LogicalVariable v : varLogical) {
Object type = env.getVarType(v);
- return nkcfProvider.getNormalizedKeyComputerFactory(type, OrderKind.ASC);
+ return nkcfProvider.getNormalizedKeyComputerFactory(type, true);
}
return null;
}
@@ -132,6 +144,18 @@
}
return typeTraits;
}
+
+ public static ITypeTraits[] variablesToTypeTraits(
+ List<LogicalVariable> varLogical, int start, int size, IVariableTypeEnvironment env, JobGenContext context)
+ throws AlgebricksException {
+ ITypeTraits[] typeTraits = new ITypeTraits[size];
+ ITypeTraitProvider typeTraitProvider = context.getTypeTraitProvider();
+ for (int i = 0; i < size; i++) {
+ Object type = env.getVarType(varLogical.get(start + i));
+ typeTraits[i] = typeTraitProvider.getTypeTrait(type);
+ }
+ return typeTraits;
+ }
public static int[] projectAllVariables(IOperatorSchema opSchema) {
int[] projectionList = new int[opSchema.getSize()];
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/OperatorSchemaImpl.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/OperatorSchemaImpl.java
similarity index 96%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/OperatorSchemaImpl.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/OperatorSchemaImpl.java
index 1d874f3..d8fc436 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/OperatorSchemaImpl.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/OperatorSchemaImpl.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl;
+package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
import java.util.ArrayList;
import java.util.HashMap;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/PlanCompiler.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java
similarity index 90%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/PlanCompiler.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java
index ab170c9..bbe8fb3 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/PlanCompiler.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl;
+package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
import java.util.ArrayList;
import java.util.HashMap;
@@ -22,11 +22,12 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.api.job.JobSpecification;
public class PlanCompiler {
@@ -52,10 +53,11 @@
reviseEdges(builder);
operatorVisitedToParents.clear();
builder.buildSpec(rootOps);
+ spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
return spec;
}
- private void compileOpRef(Mutable<ILogicalOperator> opRef, JobSpecification spec, IHyracksJobBuilder builder,
+ private void compileOpRef(Mutable<ILogicalOperator> opRef, IOperatorDescriptorRegistry spec, IHyracksJobBuilder builder,
IOperatorSchema outerPlanSchema) throws AlgebricksException {
ILogicalOperator op = opRef.getValue();
int n = op.getInputs().size();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
index 1293d5b..3382c6e 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
@@ -15,9 +15,11 @@
package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
import java.util.Collection;
+import java.util.logging.Level;
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -25,7 +27,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
public abstract class AbstractRuleController {
@@ -61,19 +62,31 @@
return rewriteOperatorRef(opRef, rule, true, false);
}
- private void printRuleApplication(IAlgebraicRewriteRule rule, Mutable<ILogicalOperator> opRef)
+ private String getPlanString(Mutable<ILogicalOperator> opRef) throws AlgebricksException {
+ if (AlgebricksConfig.ALGEBRICKS_LOGGER.isLoggable(Level.FINE)) {
+ StringBuilder sb = new StringBuilder();
+ PlanPrettyPrinter.printOperator((AbstractLogicalOperator) opRef.getValue(), sb, pvisitor, 0);
+ return sb.toString();
+ }
+ return null;
+ }
+
+ private void printRuleApplication(IAlgebraicRewriteRule rule, String beforePlan, String afterPlan)
throws AlgebricksException {
- AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Rule " + rule.getClass() + " fired.\n");
- StringBuilder sb = new StringBuilder();
- PlanPrettyPrinter.printOperator((AbstractLogicalOperator) opRef.getValue(), sb, pvisitor, 0);
- AlgebricksConfig.ALGEBRICKS_LOGGER.fine(sb.toString());
+ if (AlgebricksConfig.ALGEBRICKS_LOGGER.isLoggable(Level.FINE)) {
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Rule " + rule.getClass() + " fired.\n");
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Before plan\n" + beforePlan + "\n");
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> After plan\n" + afterPlan + "\n");
+ }
}
protected boolean rewriteOperatorRef(Mutable<ILogicalOperator> opRef, IAlgebraicRewriteRule rule,
boolean enterNestedPlans, boolean fullDFS) throws AlgebricksException {
+ String preBeforePlan = getPlanString(opRef);
if (rule.rewritePre(opRef, context)) {
- printRuleApplication(rule, opRef);
+ String preAfterPlan = getPlanString(opRef);
+ printRuleApplication(rule, preBeforePlan, preAfterPlan);
return true;
}
boolean rewritten = false;
@@ -105,8 +118,10 @@
}
}
+ String postBeforePlan = getPlanString(opRef);
if (rule.rewritePost(opRef, context)) {
- printRuleApplication(rule, opRef);
+ String postAfterPlan = getPlanString(opRef);
+ printRuleApplication(rule, postBeforePlan, postAfterPlan);
return true;
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java
index ab436f4..7c63e01 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java
@@ -20,6 +20,7 @@
import java.util.List;
import java.util.Map;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -33,7 +34,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class AlgebricksOptimizationContext implements IOptimizationContext {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
index 381f70c..471b71c 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
@@ -4,6 +4,8 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -12,9 +14,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
public class HeuristicOptimizer {
@@ -68,7 +68,7 @@
runPhysicalOptimizations(plan, physicalRewrites);
StringBuilder sb2 = new StringBuilder();
PlanPrettyPrinter.printPlan(plan, sb2, ppvisitor, 0);
- AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Optimized Plan:\n" + sb2.toString());
+ AlgebricksConfig.ALGEBRICKS_LOGGER.info("Optimized Plan:\n" + sb2.toString());
}
private void runOptimizationSets(ILogicalPlan plan,
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java
index bfbd636..fc39d0b 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java
@@ -16,9 +16,9 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public interface IAlgebraicRewriteRule {
public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-data/pom.xml b/hyracks/hyracks-algebricks/hyracks-algebricks-data/pom.xml
new file mode 100644
index 0000000..083ef45
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/pom.xml
@@ -0,0 +1,36 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-algebricks-data</artifactId>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks-common</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-data-std</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IAWriter.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IAWriter.java
new file mode 100644
index 0000000..ca79526
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IAWriter.java
@@ -0,0 +1,11 @@
+package edu.uci.ics.hyracks.algebricks.data;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+
+public interface IAWriter {
+
+ public void init() throws AlgebricksException;
+
+ public void printTuple(IFrameTupleAccessor tAccess, int tIdx) throws AlgebricksException;
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IAWriterFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IAWriterFactory.java
similarity index 84%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IAWriterFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IAWriterFactory.java
index 200f58f..71f52a6 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IAWriterFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IAWriterFactory.java
@@ -1,4 +1,4 @@
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.data;
import java.io.PrintStream;
import java.io.Serializable;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryBooleanInspector.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryBooleanInspector.java
similarity index 82%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryBooleanInspector.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryBooleanInspector.java
index 60e5050..a1b7086 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryBooleanInspector.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryBooleanInspector.java
@@ -12,10 +12,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.data;
-import java.io.Serializable;
-
-public interface IBinaryBooleanInspector extends Serializable {
+public interface IBinaryBooleanInspector {
public boolean getBooleanValue(byte[] bytes, int offset, int length);
-}
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryBooleanInspectorFactory.java
similarity index 71%
copy from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
copy to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryBooleanInspectorFactory.java
index 871fe43..23d96ca 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryBooleanInspectorFactory.java
@@ -12,10 +12,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.data;
import java.io.Serializable;
-public interface IPrinterFactory extends Serializable {
- public IPrinter createPrinter();
-}
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+
+public interface IBinaryBooleanInspectorFactory extends Serializable {
+ public IBinaryBooleanInspector createBinaryBooleanInspector(IHyracksTaskContext ctx);
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryComparatorFactoryProvider.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryComparatorFactoryProvider.java
similarity index 69%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryComparatorFactoryProvider.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryComparatorFactoryProvider.java
index bb4b734..3841e17 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryComparatorFactoryProvider.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryComparatorFactoryProvider.java
@@ -12,17 +12,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.data;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
public interface IBinaryComparatorFactoryProvider {
- /**
- * only ASC and DESC are currently supported for orderKind (FUNCTIONCALL can
- * be ignored)
- */
- public IBinaryComparatorFactory getBinaryComparatorFactory(Object type, OrderKind orderKind)
+ public IBinaryComparatorFactory getBinaryComparatorFactory(Object type, boolean ascending)
throws AlgebricksException;
}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryHashFunctionFactoryProvider.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryHashFunctionFactoryProvider.java
similarity index 86%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryHashFunctionFactoryProvider.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryHashFunctionFactoryProvider.java
index 2e1f0fa..dd7288d 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryHashFunctionFactoryProvider.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryHashFunctionFactoryProvider.java
@@ -12,9 +12,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.data;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
public interface IBinaryHashFunctionFactoryProvider {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryIntegerInspector.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryIntegerInspector.java
similarity index 82%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryIntegerInspector.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryIntegerInspector.java
index 3a2fe29..24a7eb1 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryIntegerInspector.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryIntegerInspector.java
@@ -12,10 +12,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.data;
-import java.io.Serializable;
-
-public interface IBinaryIntegerInspector extends Serializable {
+public interface IBinaryIntegerInspector {
public int getIntegerValue(byte[] bytes, int offset, int length);
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryIntegerInspectorFactory.java
similarity index 71%
copy from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
copy to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryIntegerInspectorFactory.java
index 871fe43..8760992 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryIntegerInspectorFactory.java
@@ -12,10 +12,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.data;
import java.io.Serializable;
-public interface IPrinterFactory extends Serializable {
- public IPrinter createPrinter();
-}
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+
+public interface IBinaryIntegerInspectorFactory extends Serializable {
+ public IBinaryIntegerInspector createBinaryIntegerInspector(IHyracksTaskContext ctx);
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/INormalizedKeyComputerFactoryProvider.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/INormalizedKeyComputerFactoryProvider.java
similarity index 80%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/INormalizedKeyComputerFactoryProvider.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/INormalizedKeyComputerFactoryProvider.java
index 1883ef5..6be0ec0 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/INormalizedKeyComputerFactoryProvider.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/INormalizedKeyComputerFactoryProvider.java
@@ -12,11 +12,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.data;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
public interface INormalizedKeyComputerFactoryProvider {
- public INormalizedKeyComputerFactory getNormalizedKeyComputerFactory(Object type, OrderKind order);
+ public INormalizedKeyComputerFactory getNormalizedKeyComputerFactory(Object type, boolean ascending);
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinter.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinter.java
similarity index 85%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinter.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinter.java
index 89de415..c3f42fb 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinter.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinter.java
@@ -12,11 +12,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.data;
import java.io.PrintStream;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
public interface IPrinter {
public void init() throws AlgebricksException;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinterFactory.java
similarity index 92%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinterFactory.java
index 871fe43..1a3e0d3 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinterFactory.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.data;
import java.io.Serializable;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactoryProvider.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinterFactoryProvider.java
similarity index 84%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactoryProvider.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinterFactoryProvider.java
index e2e2f1c..45a5088 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactoryProvider.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinterFactoryProvider.java
@@ -12,9 +12,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.data;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
public interface IPrinterFactoryProvider {
public IPrinterFactory getPrinterFactory(Object type) throws AlgebricksException;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/ISerializerDeserializerProvider.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ISerializerDeserializerProvider.java
similarity index 86%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/ISerializerDeserializerProvider.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ISerializerDeserializerProvider.java
index 66d696c..fdd76a0 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/ISerializerDeserializerProvider.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ISerializerDeserializerProvider.java
@@ -12,9 +12,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.data;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
public interface ISerializerDeserializerProvider {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/ITypeTraitProvider.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ITypeTraitProvider.java
similarity index 92%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/ITypeTraitProvider.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ITypeTraitProvider.java
index 6ad3079..f94caea 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/ITypeTraitProvider.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ITypeTraitProvider.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.data;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/BinaryBooleanInspectorImpl.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/BinaryBooleanInspectorImpl.java
similarity index 60%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/BinaryBooleanInspectorImpl.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/BinaryBooleanInspectorImpl.java
index fdb3663..b9bbb7d 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/BinaryBooleanInspectorImpl.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/BinaryBooleanInspectorImpl.java
@@ -12,15 +12,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data;
+package edu.uci.ics.hyracks.algebricks.data.impl;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryBooleanInspector;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspector;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
public class BinaryBooleanInspectorImpl implements IBinaryBooleanInspector {
+ public static final IBinaryBooleanInspectorFactory FACTORY = new IBinaryBooleanInspectorFactory() {
+ private static final long serialVersionUID = 1L;
- private static final long serialVersionUID = 1L;
-
- public static final BinaryBooleanInspectorImpl INSTANCE = new BinaryBooleanInspectorImpl();
+ @Override
+ public IBinaryBooleanInspector createBinaryBooleanInspector(IHyracksTaskContext ctx) {
+ return new BinaryBooleanInspectorImpl();
+ }
+ };
private BinaryBooleanInspectorImpl() {
}
@@ -29,5 +35,4 @@
public boolean getBooleanValue(byte[] bytes, int offset, int length) {
return bytes[offset] == 1;
}
-
-}
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/BinaryIntegerInspectorImpl.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/BinaryIntegerInspectorImpl.java
new file mode 100644
index 0000000..2fc170f
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/BinaryIntegerInspectorImpl.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data.impl;
+
+import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspector;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspectorFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+
+public class BinaryIntegerInspectorImpl implements IBinaryIntegerInspector {
+ public static final IBinaryIntegerInspectorFactory FACTORY = new IBinaryIntegerInspectorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IBinaryIntegerInspector createBinaryIntegerInspector(IHyracksTaskContext ctx) {
+ return new BinaryIntegerInspectorImpl();
+ }
+ };
+
+ private BinaryIntegerInspectorImpl() {
+ }
+
+ @Override
+ public int getIntegerValue(byte[] bytes, int offset, int length) {
+ return IntegerPointable.getInteger(bytes, offset);
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/IntegerPrinterFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/IntegerPrinterFactory.java
similarity index 73%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/IntegerPrinterFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/IntegerPrinterFactory.java
index db47af6..fa9bb51 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/IntegerPrinterFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/IntegerPrinterFactory.java
@@ -12,16 +12,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data;
+package edu.uci.ics.hyracks.algebricks.data.impl;
import java.io.IOException;
import java.io.PrintStream;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinter;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.WriteValueTools;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IPrinter;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.data.utils.WriteValueTools;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
public class IntegerPrinterFactory implements IPrinterFactory {
@@ -37,7 +37,7 @@
@Override
public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
- int d = IntegerSerializerDeserializer.getInt(b, s);
+ int d = IntegerPointable.getInteger(b, s);
try {
WriteValueTools.writeInt(d, ps);
} catch (IOException e) {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/NoopNullWriterFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/NoopNullWriterFactory.java
similarity index 94%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/NoopNullWriterFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/NoopNullWriterFactory.java
index e064832..8011091 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/NoopNullWriterFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/NoopNullWriterFactory.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data;
+package edu.uci.ics.hyracks.algebricks.data.impl;
import java.io.DataOutput;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/UTF8StringPrinterFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/UTF8StringPrinterFactory.java
similarity index 83%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/UTF8StringPrinterFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/UTF8StringPrinterFactory.java
index 4a900f1..139866a 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/UTF8StringPrinterFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/UTF8StringPrinterFactory.java
@@ -12,13 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data;
+package edu.uci.ics.hyracks.algebricks.data.impl;
import java.io.PrintStream;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinter;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IPrinter;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
public class UTF8StringPrinterFactory implements IPrinterFactory {
@@ -36,7 +36,7 @@
@Override
public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
- int strlen = UTF8StringPointable.getUTFLen(b, s);
+ int strlen = UTF8StringPointable.getUTFLength(b, s);
int pos = s + 2;
int maxPos = pos + strlen;
ps.print("\"");
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/WriteValueTools.java b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/utils/WriteValueTools.java
similarity index 94%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/WriteValueTools.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/utils/WriteValueTools.java
index 8ecd16b..7b045dd 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/WriteValueTools.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/utils/WriteValueTools.java
@@ -1,4 +1,4 @@
-package edu.uci.ics.hyracks.algebricks.core.utils;
+package edu.uci.ics.hyracks.algebricks.data.utils;
import java.io.IOException;
import java.io.OutputStream;
@@ -58,7 +58,7 @@
}
public static void writeUTF8String(byte[] b, int s, int l, OutputStream os) throws IOException {
- int stringLength = UTF8StringPointable.getUTFLen(b, s);
+ int stringLength = UTF8StringPointable.getUTFLength(b, s);
int position = s + 2;
int maxPosition = position + stringLength;
os.write('\"');
@@ -82,7 +82,7 @@
}
public static void writeUTF8StringNoQuotes(byte[] b, int s, int l, OutputStream os) throws IOException {
- int stringLength = UTF8StringPointable.getUTFLen(b, s);
+ int stringLength = UTF8StringPointable.getUTFLength(b, s);
int position = s + 2;
int maxPosition = position + stringLength;
while (position < maxPosition) {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java
index 07f6f5d..d105759 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java
@@ -12,6 +12,8 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.compiler.api.HeuristicCompilerFactoryBuilder;
import edu.uci.ics.hyracks.algebricks.compiler.api.ICompiler;
import edu.uci.ics.hyracks.algebricks.compiler.api.ICompilerFactory;
@@ -21,10 +23,10 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.ISerializerDeserializerProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.LogicalExpressionJobGenToExpressionRuntimeProviderAdapter;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
@@ -38,10 +40,10 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.data.ISerializerDeserializerProvider;
+import edu.uci.ics.hyracks.algebricks.data.ITypeTraitProvider;
import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.ASTNode;
import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.AssignmentNode;
import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.DumpNode;
@@ -64,6 +66,7 @@
import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Schema;
import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.job.JobSpecification;
public class PigletCompiler {
@@ -123,8 +126,14 @@
return null;
}
});
+ builder.setTypeTraitProvider(new ITypeTraitProvider() {
+ public ITypeTraits getTypeTrait(Object type) {
+ return null;
+ }
+ });
builder.setPrinterProvider(PigletPrinterFactoryProvider.INSTANCE);
- builder.setExprJobGen(new PigletExpressionJobGen());
+ builder.setExpressionRuntimeProvider(new LogicalExpressionJobGenToExpressionRuntimeProviderAdapter(
+ new PigletExpressionJobGen()));
builder.setExpressionTypeComputer(new IExpressionTypeComputer() {
@Override
public Object getType(ILogicalExpression expr, IMetadataProvider<?, ?> metadataProvider,
@@ -294,22 +303,22 @@
private IFunctionInfo lookupFunction(FunctionTag functionTag, String functionName) throws PigletException {
switch (functionTag) {
case EQ:
- return AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.EQ);
+ return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.EQ);
case NEQ:
- return AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.NEQ);
+ return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.NEQ);
case LT:
- return AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.LT);
+ return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.LT);
case LTE:
- return AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.LE);
+ return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.LE);
case GT:
- return AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.GT);
+ return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.GT);
case GTE:
- return AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.GE);
+ return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.GE);
}
throw new PigletException("Unsupported function: " + functionTag);
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletPrinterFactoryProvider.java b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletPrinterFactoryProvider.java
index cb12b7d..95797bc 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletPrinterFactoryProvider.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletPrinterFactoryProvider.java
@@ -3,12 +3,12 @@
import java.io.IOException;
import java.io.PrintStream;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinter;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data.IntegerPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.WriteValueTools;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IPrinter;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.impl.IntegerPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.data.utils.WriteValueTools;
import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionInfoImpl.java b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFunction.java
similarity index 70%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionInfoImpl.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFunction.java
index 15521d6..439ed7a 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionInfoImpl.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFunction.java
@@ -12,13 +12,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.functions;
+package edu.uci.ics.hyracks.algebricks.examples.piglet.metadata;
-public class FunctionInfoImpl implements IFunctionInfo {
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+public class PigletFunction implements IFunctionInfo {
private final FunctionIdentifier fid;
- public FunctionInfoImpl(FunctionIdentifier fid) {
+ public PigletFunction(FunctionIdentifier fid) {
this.fid = fid;
}
@@ -26,10 +28,4 @@
public FunctionIdentifier getFunctionIdentifier() {
return fid;
}
-
- @Override
- public Object getInfo() {
- throw new IllegalStateException();
- }
-
-}
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
index 668d7a2..d678803 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
@@ -1,23 +1,31 @@
package edu.uci.ics.hyracks.algebricks.examples.piglet.metadata;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.SinkWriterRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.writers.PrinterBasedWriterFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksAbsolutePartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.SinkWriterRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.writers.PrinterBasedWriterFactory;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
@@ -37,6 +45,16 @@
import edu.uci.ics.hyracks.dataflow.std.file.ITupleParserFactory;
public class PigletMetadataProvider implements IMetadataProvider<String, String> {
+ private static final Map<FunctionIdentifier, PigletFunction> FN_MAP;
+
+ static {
+ Map<FunctionIdentifier, PigletFunction> map = new HashMap<FunctionIdentifier, PigletFunction>();
+
+ map.put(AlgebricksBuiltinFunctions.EQ, new PigletFunction(AlgebricksBuiltinFunctions.EQ));
+
+ FN_MAP = Collections.unmodifiableMap(map);
+ }
+
@Override
public IDataSource<String> findDataSource(String id) throws AlgebricksException {
return null;
@@ -44,9 +62,13 @@
@SuppressWarnings("unchecked")
@Override
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<String> dataSource,
- List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
- JobGenContext context, JobSpecification jobSpec) throws AlgebricksException {
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(
+ IDataSource<String> dataSource,
+ List<LogicalVariable> scanVariables,
+ List<LogicalVariable> projectVariables, boolean projectPushed,
+ IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv,
+ JobGenContext context, JobSpecification jobSpec)
+ throws AlgebricksException {
PigletFileDataSource ds = (PigletFileDataSource) dataSource;
FileSplit[] fileSplits = ds.getFileSplits();
@@ -149,7 +171,8 @@
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexInsertRuntime(
IDataSourceIndex<String, String> dataSource, IOperatorSchema propagatedSchema,
- List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys, RecordDescriptor recordDesc,
+ IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys,
+ List<LogicalVariable> secondaryKeys, ILogicalExpression filterExpr, RecordDescriptor recordDesc,
JobGenContext context, JobSpecification spec) throws AlgebricksException {
// TODO Auto-generated method stub
return null;
@@ -158,10 +181,15 @@
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexDeleteRuntime(
IDataSourceIndex<String, String> dataSource, IOperatorSchema propagatedSchema,
- List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys, RecordDescriptor recordDesc,
+ IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys,
+ List<LogicalVariable> secondaryKeys, ILogicalExpression filterExpr, RecordDescriptor recordDesc,
JobGenContext context, JobSpecification spec) throws AlgebricksException {
// TODO Auto-generated method stub
return null;
}
-
-}
\ No newline at end of file
+
+ @Override
+ public IFunctionInfo lookupFunction(FunctionIdentifier fid) {
+ return FN_MAP.get(fid);
+ }
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/rewriter/PigletRewriteRuleset.java b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/rewriter/PigletRewriteRuleset.java
index 90ee5bb..b2a5215 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/rewriter/PigletRewriteRuleset.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/rewriter/PigletRewriteRuleset.java
@@ -16,7 +16,6 @@
import edu.uci.ics.hyracks.algebricks.rewriter.rules.FactorRedundantGroupAndDecorVarsRule;
import edu.uci.ics.hyracks.algebricks.rewriter.rules.InferTypesRule;
import edu.uci.ics.hyracks.algebricks.rewriter.rules.InlineVariablesRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.IntroduceGroupByForStandaloneAggregRule;
import edu.uci.ics.hyracks.algebricks.rewriter.rules.IsolateHyracksOperatorsRule;
import edu.uci.ics.hyracks.algebricks.rewriter.rules.PullSelectOutOfEqJoin;
import edu.uci.ics.hyracks.algebricks.rewriter.rules.PushLimitDownRule;
@@ -40,7 +39,10 @@
public final static List<IAlgebraicRewriteRule> buildNormalizationRuleCollection() {
List<IAlgebraicRewriteRule> normalization = new LinkedList<IAlgebraicRewriteRule>();
normalization.add(new EliminateSubplanRule());
- normalization.add(new IntroduceGroupByForStandaloneAggregRule());
+ // TODO: This rule is incorrect and has been removed. Its replacement in
+ // Asterix (PushAggFuncIntoStandaloneAggregateRule)
+ // is language-specific.
+ // normalization.add(new IntroduceGroupByForStandaloneAggregRule());
normalization.add(new BreakSelectIntoConjunctsRule());
normalization.add(new PushSelectIntoJoinRule());
normalization.add(new ExtractGbyExpressionsRule());
@@ -100,7 +102,6 @@
return physicalPlanRewrites;
}
-
public final static List<IAlgebraicRewriteRule> prepareForJobGenRuleCollection() {
List<IAlgebraicRewriteRule> prepareForJobGenRewrites = new LinkedList<IAlgebraicRewriteRule>();
prepareForJobGenRewrites.add(new IsolateHyracksOperatorsRule(
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java
index 63e6f53..08947d1 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java
@@ -6,6 +6,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
@@ -17,26 +18,25 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IRunningAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.ISerializableAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IUnnestingFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.evaluators.ColumnAccessEvalFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.evaluators.ConstantEvalFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import edu.uci.ics.hyracks.algebricks.examples.piglet.compiler.ConstantValue;
import edu.uci.ics.hyracks.algebricks.examples.piglet.exceptions.PigletException;
import edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.functions.PigletFunctionRegistry;
import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyRunningAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyUnnestingFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.evaluators.ColumnAccessEvalFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.evaluators.ConstantEvalFactory;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
public class PigletExpressionJobGen implements ILogicalExpressionJobGen {
@Override
- public IEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
+ public ICopyEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException {
switch (expr.getExpressionTag()) {
case CONSTANT: {
@@ -65,19 +65,19 @@
default:
throw new UnsupportedOperationException("Unsupported constant type: " + type.getTag());
}
- return new ConstantEvalFactory(Arrays.copyOf(abvs.getBytes(), abvs.getLength()));
+ return new ConstantEvalFactory(Arrays.copyOf(abvs.getByteArray(), abvs.getLength()));
}
case FUNCTION_CALL: {
ScalarFunctionCallExpression sfce = (ScalarFunctionCallExpression) expr;
List<Mutable<ILogicalExpression>> argExprs = sfce.getArguments();
- IEvaluatorFactory argEvalFactories[] = new IEvaluatorFactory[argExprs.size()];
+ ICopyEvaluatorFactory argEvalFactories[] = new ICopyEvaluatorFactory[argExprs.size()];
for (int i = 0; i < argEvalFactories.length; ++i) {
Mutable<ILogicalExpression> er = argExprs.get(i);
argEvalFactories[i] = createEvaluatorFactory(er.getValue(), env, inputSchemas, context);
}
- IEvaluatorFactory funcEvalFactory;
+ ICopyEvaluatorFactory funcEvalFactory;
try {
funcEvalFactory = PigletFunctionRegistry.createFunctionEvaluatorFactory(
sfce.getFunctionIdentifier(), argEvalFactories);
@@ -97,28 +97,28 @@
}
@Override
- public IAggregateFunctionFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr,
+ public ICopyAggregateFunctionFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr,
IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
throws AlgebricksException {
throw new UnsupportedOperationException();
}
@Override
- public ISerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory(
+ public ICopySerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory(
AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas,
JobGenContext context) throws AlgebricksException {
throw new UnsupportedOperationException();
}
@Override
- public IRunningAggregateFunctionFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr,
+ public ICopyRunningAggregateFunctionFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr,
IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
throws AlgebricksException {
throw new UnsupportedOperationException();
}
@Override
- public IUnnestingFunctionFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr,
+ public ICopyUnnestingFunctionFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr,
IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
throws AlgebricksException {
throw new UnsupportedOperationException();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IPigletFunctionEvaluatorFactoryBuilder.java b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IPigletFunctionEvaluatorFactoryBuilder.java
index 38e7615..7aa3734 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IPigletFunctionEvaluatorFactoryBuilder.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IPigletFunctionEvaluatorFactoryBuilder.java
@@ -1,8 +1,8 @@
package edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.functions;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
public interface IPigletFunctionEvaluatorFactoryBuilder {
- public IEvaluatorFactory buildEvaluatorFactory(FunctionIdentifier fid, IEvaluatorFactory[] arguments);
+ public ICopyEvaluatorFactory buildEvaluatorFactory(FunctionIdentifier fid, ICopyEvaluatorFactory[] arguments);
}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IntegerEqFunctionEvaluatorFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IntegerEqFunctionEvaluatorFactory.java
index d654d77..57706a1 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IntegerEqFunctionEvaluatorFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IntegerEqFunctionEvaluatorFactory.java
@@ -3,34 +3,34 @@
import java.io.DataOutput;
import java.io.IOException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-public class IntegerEqFunctionEvaluatorFactory implements IEvaluatorFactory {
+public class IntegerEqFunctionEvaluatorFactory implements ICopyEvaluatorFactory {
private static final long serialVersionUID = 1L;
- private final IEvaluatorFactory arg1Factory;
+ private final ICopyEvaluatorFactory arg1Factory;
- private final IEvaluatorFactory arg2Factory;
+ private final ICopyEvaluatorFactory arg2Factory;
- public IntegerEqFunctionEvaluatorFactory(IEvaluatorFactory arg1Factory, IEvaluatorFactory arg2Factory) {
+ public IntegerEqFunctionEvaluatorFactory(ICopyEvaluatorFactory arg1Factory, ICopyEvaluatorFactory arg2Factory) {
this.arg1Factory = arg1Factory;
this.arg2Factory = arg2Factory;
}
@Override
- public IEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
- return new IEvaluator() {
+ public ICopyEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
+ return new ICopyEvaluator() {
private DataOutput dataout = output.getDataOutput();
private ArrayBackedValueStorage out1 = new ArrayBackedValueStorage();
private ArrayBackedValueStorage out2 = new ArrayBackedValueStorage();
- private IEvaluator eval1 = arg1Factory.createEvaluator(out1);
- private IEvaluator eval2 = arg2Factory.createEvaluator(out2);
+ private ICopyEvaluator eval1 = arg1Factory.createEvaluator(out1);
+ private ICopyEvaluator eval2 = arg2Factory.createEvaluator(out2);
@Override
public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
@@ -38,8 +38,8 @@
eval1.evaluate(tuple);
out2.reset();
eval2.evaluate(tuple);
- int v1 = IntegerSerializerDeserializer.getInt(out1.getBytes(), 0);
- int v2 = IntegerSerializerDeserializer.getInt(out2.getBytes(), 0);
+ int v1 = IntegerSerializerDeserializer.getInt(out1.getByteArray(), 0);
+ int v2 = IntegerSerializerDeserializer.getInt(out2.getByteArray(), 0);
boolean r = v1 == v2;
try {
dataout.writeBoolean(r);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/PigletFunctionRegistry.java b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/PigletFunctionRegistry.java
index 5acded6..bf37c0b 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/PigletFunctionRegistry.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/PigletFunctionRegistry.java
@@ -6,8 +6,8 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
import edu.uci.ics.hyracks.algebricks.examples.piglet.exceptions.PigletException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
public class PigletFunctionRegistry {
private static final Map<FunctionIdentifier, IPigletFunctionEvaluatorFactoryBuilder> builderMap;
@@ -17,7 +17,7 @@
temp.put(AlgebricksBuiltinFunctions.EQ, new IPigletFunctionEvaluatorFactoryBuilder() {
@Override
- public IEvaluatorFactory buildEvaluatorFactory(FunctionIdentifier fid, IEvaluatorFactory[] arguments) {
+ public ICopyEvaluatorFactory buildEvaluatorFactory(FunctionIdentifier fid, ICopyEvaluatorFactory[] arguments) {
return new IntegerEqFunctionEvaluatorFactory(arguments[0], arguments[1]);
}
});
@@ -25,7 +25,7 @@
builderMap = Collections.unmodifiableMap(temp);
}
- public static IEvaluatorFactory createFunctionEvaluatorFactory(FunctionIdentifier fid, IEvaluatorFactory[] args)
+ public static ICopyEvaluatorFactory createFunctionEvaluatorFactory(FunctionIdentifier fid, ICopyEvaluatorFactory[] args)
throws PigletException {
IPigletFunctionEvaluatorFactoryBuilder builder = builderMap.get(fid);
if (builder == null) {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Schema.java b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Schema.java
index ba2401c..8d6e559 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Schema.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Schema.java
@@ -2,7 +2,7 @@
import java.util.List;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
public class Schema {
private List<Pair<String, Type>> schema;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/javacc/PigletParser.jj b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/javacc/PigletParser.jj
index 9e339e4..2541536 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/javacc/PigletParser.jj
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/javacc/PigletParser.jj
@@ -23,7 +23,7 @@
package edu.uci.ics.hyracks.algebricks.examples.piglet.parser;
import java.util.*;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.*;
import edu.uci.ics.hyracks.algebricks.examples.piglet.types.*;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
index fe33d09..3b7fa8c 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
@@ -22,6 +22,8 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -33,9 +35,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
import edu.uci.ics.hyracks.algebricks.rewriter.util.PhysicalOptimizationsUtil;
public abstract class AbstractDecorrelationRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java
index fbacede..553165a 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java
@@ -17,13 +17,13 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public abstract class AbstractExtractExprRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractIntroduceCombinerRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractIntroduceCombinerRule.java
new file mode 100644
index 0000000..08271c1
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractIntroduceCombinerRule.java
@@ -0,0 +1,144 @@
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public abstract class AbstractIntroduceCombinerRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ /**
+ * Replace the original aggregate functions with their corresponding global aggregate function.
+ */
+ public void replaceOriginalAggFuncs(Map<AggregateFunctionCallExpression, SimilarAggregatesInfo> toReplaceMap) {
+ for (Map.Entry<AggregateFunctionCallExpression, SimilarAggregatesInfo> entry : toReplaceMap.entrySet()) {
+ SimilarAggregatesInfo sai = entry.getValue();
+ for (AggregateExprInfo aei : sai.simAggs) {
+ AbstractFunctionCallExpression afce = (AbstractFunctionCallExpression) aei.aggExprRef.getValue();
+ afce.setFunctionInfo(aei.newFunInfo);
+ afce.getArguments().clear();
+ afce.getArguments().add(new MutableObject<ILogicalExpression>(sai.stepOneResult));
+ }
+ }
+ }
+
+ protected Pair<Boolean, Mutable<ILogicalOperator>> tryToPushAgg(AggregateOperator initAgg,
+ GroupByOperator newGbyOp, Map<AggregateFunctionCallExpression, SimilarAggregatesInfo> toReplaceMap,
+ IOptimizationContext context) throws AlgebricksException {
+
+ ArrayList<LogicalVariable> pushedVars = new ArrayList<LogicalVariable>();
+ ArrayList<Mutable<ILogicalExpression>> pushedExprs = new ArrayList<Mutable<ILogicalExpression>>();
+
+ List<LogicalVariable> initVars = initAgg.getVariables();
+ List<Mutable<ILogicalExpression>> initExprs = initAgg.getExpressions();
+ int numExprs = initVars.size();
+
+ // First make sure that all agg funcs are two step, otherwise we cannot use local aggs.
+ for (int i = 0; i < numExprs; i++) {
+ AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) initExprs.get(i).getValue();
+ if (!aggFun.isTwoStep()) {
+ return new Pair<Boolean, Mutable<ILogicalOperator>>(false, null);
+ }
+ }
+
+ boolean haveAggToReplace = false;
+ for (int i = 0; i < numExprs; i++) {
+ Mutable<ILogicalExpression> expRef = initExprs.get(i);
+ AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) expRef.getValue();
+ IFunctionInfo fi1 = aggFun.getStepOneAggregate();
+ // Clone the aggregate's args.
+ List<Mutable<ILogicalExpression>> newArgs = new ArrayList<Mutable<ILogicalExpression>>(aggFun
+ .getArguments().size());
+ for (Mutable<ILogicalExpression> er : aggFun.getArguments()) {
+ newArgs.add(new MutableObject<ILogicalExpression>(er.getValue().cloneExpression()));
+ }
+ IFunctionInfo fi2 = aggFun.getStepTwoAggregate();
+ SimilarAggregatesInfo inf = toReplaceMap.get(aggFun);
+ if (inf == null) {
+ inf = new SimilarAggregatesInfo();
+ LogicalVariable newAggVar = context.newVar();
+ pushedVars.add(newAggVar);
+ inf.stepOneResult = new VariableReferenceExpression(newAggVar);
+ inf.simAggs = new ArrayList<AggregateExprInfo>();
+ toReplaceMap.put(aggFun, inf);
+ AggregateFunctionCallExpression aggLocal = new AggregateFunctionCallExpression(fi1, false, newArgs);
+ pushedExprs.add(new MutableObject<ILogicalExpression>(aggLocal));
+ }
+ AggregateExprInfo aei = new AggregateExprInfo();
+ aei.aggExprRef = expRef;
+ aei.newFunInfo = fi2;
+ inf.simAggs.add(aei);
+ haveAggToReplace = true;
+ }
+
+ if (!pushedVars.isEmpty()) {
+ AggregateOperator pushedAgg = new AggregateOperator(pushedVars, pushedExprs);
+ pushedAgg.setExecutionMode(ExecutionMode.LOCAL);
+ // If newGbyOp is null, then we optimizing an aggregate without group by.
+ if (newGbyOp != null) {
+ // Hook up the nested aggregate op with the outer group by.
+ NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(
+ newGbyOp));
+ nts.setExecutionMode(ExecutionMode.LOCAL);
+ pushedAgg.getInputs().add(new MutableObject<ILogicalOperator>(nts));
+ } else {
+ // The local aggregate operator is fed by the input of the original aggregate operator.
+ pushedAgg.getInputs().add(new MutableObject<ILogicalOperator>(initAgg.getInputs().get(0).getValue()));
+ // Set the partitioning variable in the local agg to ensure it is not projected away.
+ context.computeAndSetTypeEnvironmentForOperator(pushedAgg);
+ LogicalVariable trueVar = context.newVar();
+ // Reintroduce assign op for the global agg partitioning var.
+ AssignOperator trueAssignOp = new AssignOperator(trueVar, new MutableObject<ILogicalExpression>(
+ ConstantExpression.TRUE));
+ trueAssignOp.getInputs().add(new MutableObject<ILogicalOperator>(pushedAgg));
+ context.computeAndSetTypeEnvironmentForOperator(trueAssignOp);
+ initAgg.setPartitioningVariable(trueVar);
+ initAgg.getInputs().get(0).setValue(trueAssignOp);
+ }
+ return new Pair<Boolean, Mutable<ILogicalOperator>>(true, new MutableObject<ILogicalOperator>(pushedAgg));
+ } else {
+ return new Pair<Boolean, Mutable<ILogicalOperator>>(haveAggToReplace, null);
+ }
+ }
+
+ protected class SimilarAggregatesInfo {
+ ILogicalExpression stepOneResult;
+ List<AggregateExprInfo> simAggs;
+ }
+
+ protected class AggregateExprInfo {
+ Mutable<ILogicalExpression> aggExprRef;
+ IFunctionInfo newFunInfo;
+ }
+
+ protected class BookkeepingInfo {
+ Map<AggregateFunctionCallExpression, SimilarAggregatesInfo> toReplaceMap = new HashMap<AggregateFunctionCallExpression, SimilarAggregatesInfo>();
+ Map<GroupByOperator, List<LogicalVariable>> modifyGbyMap = new HashMap<GroupByOperator, List<LogicalVariable>>();
+ }
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java
index 55eef21..53b361b 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java
@@ -20,13 +20,13 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class BreakSelectIntoConjunctsRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java
index 2ac387d..4f6699a 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java
@@ -19,6 +19,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -33,7 +34,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class ComplexJoinInferenceRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexUnnestToProductRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexUnnestToProductRule.java
new file mode 100644
index 0000000..4521d1a
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexUnnestToProductRule.java
@@ -0,0 +1,276 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Complex rewrite rule for producing joins from unnests.
+ * This rule is limited to creating left-deep trees.
+ */
+public class ComplexUnnestToProductRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN
+ && op.getOperatorTag() != LogicalOperatorTag.UNNEST) {
+ return false;
+ }
+
+ // We may pull selects above the join we create in order to eliminate possible dependencies between
+ // the outer and inner input plans of the join.
+ List<ILogicalOperator> topSelects = new ArrayList<ILogicalOperator>();
+
+ // Keep track of the operators and used variables participating in the inner input plan.
+ HashSet<LogicalVariable> innerUsedVars = new HashSet<LogicalVariable>();
+ List<ILogicalOperator> innerOps = new ArrayList<ILogicalOperator>();
+ HashSet<LogicalVariable> outerUsedVars = new HashSet<LogicalVariable>();
+ List<ILogicalOperator> outerOps = new ArrayList<ILogicalOperator>();
+ innerOps.add(op);
+ VariableUtilities.getUsedVariables(op, innerUsedVars);
+
+ Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+
+ // Find an unnest or join and partition the plan between the first unnest and that operator into independent parts.
+ if (!findPlanPartition(op2, innerUsedVars, outerUsedVars, innerOps, outerOps, topSelects, false)) {
+ // We could not find an unnest or join.
+ return false;
+ }
+ // The last operator must be an unnest or join.
+ AbstractLogicalOperator unnestOrJoin = (AbstractLogicalOperator) outerOps.get(outerOps.size() - 1);
+
+ ILogicalOperator outerRoot = null;
+ ILogicalOperator innerRoot = null;
+ EmptyTupleSourceOperator ets = new EmptyTupleSourceOperator();
+ // If we found a join, simply use it as the outer root.
+ if (unnestOrJoin.getOperatorTag() != LogicalOperatorTag.INNERJOIN
+ && unnestOrJoin.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
+ // We've found a second unnest. First, sanity check that the unnest does not produce any vars that are used by the plan above (until the first unnest).
+ List<LogicalVariable> producedVars = new ArrayList<LogicalVariable>();
+ VariableUtilities.getProducedVariables(unnestOrJoin, producedVars);
+ for (LogicalVariable producedVar : producedVars) {
+ if (innerUsedVars.contains(producedVar)) {
+ return false;
+ }
+ }
+ // Continue finding a partitioning of the plan such that the inner and outer partitions are independent, in order to feed a join.
+ // Now, we look below the second unnest or join.
+ VariableUtilities.getUsedVariables(unnestOrJoin, outerUsedVars);
+ AbstractLogicalOperator unnestChild = (AbstractLogicalOperator) unnestOrJoin.getInputs().get(0).getValue();
+ if (!findPlanPartition(unnestChild, innerUsedVars, outerUsedVars, innerOps, outerOps, topSelects, true)) {
+ // We could not find a suitable partitioning.
+ return false;
+ }
+ }
+ innerRoot = buildOperatorChain(innerOps, ets, context);
+ context.computeAndSetTypeEnvironmentForOperator(innerRoot);
+ outerRoot = buildOperatorChain(outerOps, null, context);
+ context.computeAndSetTypeEnvironmentForOperator(outerRoot);
+
+ InnerJoinOperator product = new InnerJoinOperator(
+ new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
+ // Outer branch.
+ product.getInputs().add(new MutableObject<ILogicalOperator>(outerRoot));
+ // Inner branch.
+ product.getInputs().add(new MutableObject<ILogicalOperator>(innerRoot));
+ context.computeAndSetTypeEnvironmentForOperator(product);
+ // Put the selects on top of the join.
+ ILogicalOperator topOp = product;
+ if (!topSelects.isEmpty()) {
+ topOp = buildOperatorChain(topSelects, product, context);
+ }
+ // Plug the selects + product in the plan.
+ opRef.setValue(topOp);
+ context.computeAndSetTypeEnvironmentForOperator(topOp);
+ return true;
+ }
+
+ private ILogicalOperator buildOperatorChain(List<ILogicalOperator> ops, ILogicalOperator bottomOp,
+ IOptimizationContext context) throws AlgebricksException {
+ ILogicalOperator root = ops.get(0);
+ ILogicalOperator prevOp = root;
+ for (int i = 1; i < ops.size(); i++) {
+ ILogicalOperator inputOp = ops.get(i);
+ prevOp.getInputs().clear();
+ prevOp.getInputs().add(new MutableObject<ILogicalOperator>(inputOp));
+ prevOp = inputOp;
+ }
+ if (bottomOp != null) {
+ context.computeAndSetTypeEnvironmentForOperator(bottomOp);
+ prevOp.getInputs().clear();
+ prevOp.getInputs().add(new MutableObject<ILogicalOperator>(bottomOp));
+ }
+ return root;
+ }
+
+ private boolean findPlanPartition(AbstractLogicalOperator op, HashSet<LogicalVariable> innerUsedVars,
+ HashSet<LogicalVariable> outerUsedVars, List<ILogicalOperator> innerOps, List<ILogicalOperator> outerOps,
+ List<ILogicalOperator> topSelects, boolean belowSecondUnnest) throws AlgebricksException {
+ if (belowSecondUnnest && innerUsedVars.isEmpty()) {
+ // Trivially joinable.
+ return true;
+ }
+ if (!belowSecondUnnest && op.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
+ // Bail on subplan.
+ return false;
+ }
+ switch (op.getOperatorTag()) {
+ case UNNEST:
+ case DATASOURCESCAN: {
+ // We may have reached this state by descending through a subplan.
+ outerOps.add(op);
+ return true;
+ }
+ case INNERJOIN:
+ case LEFTOUTERJOIN: {
+ // Make sure that no variables that are live under this join are needed by the inner.
+ List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
+ VariableUtilities.getLiveVariables(op, liveVars);
+ for (LogicalVariable liveVar : liveVars) {
+ if (innerUsedVars.contains(liveVar)) {
+ return false;
+ }
+ }
+ outerOps.add(op);
+ return true;
+ }
+ case SELECT: {
+ // Remember this select to pulling it above the join.
+ if (innerUsedVars.isEmpty()) {
+ outerOps.add(op);
+ } else {
+ topSelects.add(op);
+ }
+ break;
+ }
+ case PROJECT: {
+ // Throw away projects from the plan since we are pulling selects up.
+ break;
+ }
+ case EMPTYTUPLESOURCE:
+ case NESTEDTUPLESOURCE: {
+ if (belowSecondUnnest) {
+ // We have successfully partitioned the plan into independent parts to be plugged into the join.
+ return true;
+ } else {
+ // We could not find a second unnest or a join.
+ return false;
+ }
+ }
+ default: {
+ // The inner is trivially independent.
+ if (!belowSecondUnnest && innerUsedVars.isEmpty()) {
+ outerOps.add(op);
+ break;
+ }
+
+ // Examine produced vars to determine which partition uses them.
+ List<LogicalVariable> producedVars = new ArrayList<LogicalVariable>();
+ VariableUtilities.getProducedVariables(op, producedVars);
+ int outerMatches = 0;
+ int innerMatches = 0;
+ for (LogicalVariable producedVar : producedVars) {
+ if (outerUsedVars.contains(producedVar)) {
+ outerMatches++;
+ } else if (innerUsedVars.contains(producedVar)) {
+ innerMatches++;
+ }
+ }
+
+ HashSet<LogicalVariable> targetUsedVars = null;
+ if (outerMatches == producedVars.size() && !producedVars.isEmpty()) {
+ // All produced vars used by outer partition.
+ outerOps.add(op);
+ targetUsedVars = outerUsedVars;
+ } else if (innerMatches == producedVars.size() && !producedVars.isEmpty()) {
+ // All produced vars used by inner partition.
+ innerOps.add(op);
+ targetUsedVars = innerUsedVars;
+ } else if (innerMatches == 0 && outerMatches == 0) {
+ // Op produces variables that are not used in the part of the plan we've seen (or it doesn't produce any vars).
+ // Try to figure out where it belongs by analyzing the used variables.
+ List<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
+ VariableUtilities.getUsedVariables(op, usedVars);
+ for (LogicalVariable usedVar : usedVars) {
+ if (outerUsedVars.contains(usedVar)) {
+ outerOps.add(op);
+ targetUsedVars = outerUsedVars;
+ break;
+ }
+ if (innerUsedVars.contains(usedVar)) {
+ innerOps.add(op);
+ targetUsedVars = innerUsedVars;
+ break;
+ }
+ }
+ // TODO: For now we bail here, but we could remember such ops and determine their target partition at a later point.
+ if (targetUsedVars == null) {
+ return false;
+ }
+ } else {
+ // The current operator produces variables that are used by both partitions, so the inner and outer are not independent and, therefore, we cannot create a join.
+ // TODO: We may still be able to split the operator to create a viable partitioning.
+ return false;
+ }
+ // Update used variables of partition that op belongs to.
+ if (op.hasNestedPlans() && op.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+ AbstractOperatorWithNestedPlans opWithNestedPlans = (AbstractOperatorWithNestedPlans) op;
+ opWithNestedPlans.getUsedVariablesExceptNestedPlans(targetUsedVars);
+ } else {
+ VariableUtilities.getUsedVariables(op, targetUsedVars);
+ }
+ break;
+ }
+ }
+ if (!op.hasInputs()) {
+ if (!belowSecondUnnest) {
+ // We could not find a second unnest or a join.
+ return false;
+ } else {
+ // We have successfully partitioned the plan into independent parts to be plugged into the join.
+ return true;
+ }
+ }
+ return findPlanPartition((AbstractLogicalOperator) op.getInputs().get(0).getValue(), innerUsedVars,
+ outerUsedVars, innerOps, outerOps, topSelects, belowSecondUnnest);
+ }
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
index 67e7cf0..e0220cc 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
@@ -19,6 +19,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
@@ -26,7 +27,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class ConsolidateAssignsRule implements IAlgebraicRewriteRule {
@@ -37,7 +37,8 @@
}
@Override
- public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
return false;
@@ -48,7 +49,6 @@
if (op2.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
return false;
}
-
AssignOperator assign2 = (AssignOperator) op2;
HashSet<LogicalVariable> used1 = new HashSet<LogicalVariable>();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java
index c2bb13d..ff9d921 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java
@@ -14,11 +14,10 @@
*/
package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-import java.util.List;
-
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -26,11 +25,19 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+/**
+ * Matches the following operator pattern:
+ * (select) <-- ((assign)* <-- (select)*)+
+ *
+ * Consolidates the selects to:
+ * (select) <-- (assign)*
+ *
+ */
public class ConsolidateSelectsRule implements IAlgebraicRewriteRule {
@Override
@@ -40,39 +47,63 @@
@Override
public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
- AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
return false;
}
- SelectOperator select = (SelectOperator) op;
+ SelectOperator firstSelect = (SelectOperator) op;
- AbstractLogicalOperator op2 = (AbstractLogicalOperator) select.getInputs().get(0).getValue();
- if (op2.getOperatorTag() != LogicalOperatorTag.SELECT) {
- return false;
- }
-
- AbstractFunctionCallExpression conj = new ScalarFunctionCallExpression(
- AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.AND));
- conj.getArguments().add(new MutableObject<ILogicalExpression>(select.getCondition().getValue()));
- conj.getArguments().add(((SelectOperator) op2).getCondition());
-
- Mutable<ILogicalOperator> botOpRef = select.getInputs().get(0);
- boolean more = true;
- while (more) {
- botOpRef = botOpRef.getValue().getInputs().get(0);
- AbstractLogicalOperator botOp = (AbstractLogicalOperator) botOpRef.getValue();
- if (botOp.getOperatorTag() == LogicalOperatorTag.SELECT) {
- conj.getArguments().add(((SelectOperator) botOp).getCondition());
- } else {
- more = false;
+ IFunctionInfo andFn = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.AND);
+ // New conjuncts for consolidated select.
+ AbstractFunctionCallExpression conj = null;
+ AbstractLogicalOperator topMostOp = null;
+ AbstractLogicalOperator selectParent = null;
+ AbstractLogicalOperator nextSelect = firstSelect;
+ do {
+ // Skip through assigns.
+ do {
+ selectParent = nextSelect;
+ nextSelect = (AbstractLogicalOperator) selectParent.getInputs().get(0).getValue();
+ } while (nextSelect.getOperatorTag() == LogicalOperatorTag.ASSIGN);
+ // Stop if the child op is not a select.
+ if (nextSelect.getOperatorTag() != LogicalOperatorTag.SELECT) {
+ break;
+ }
+ // Remember the top-most op that we are not removing.
+ topMostOp = selectParent;
+
+ // Initialize the new conjuncts, if necessary.
+ if (conj == null) {
+ conj = new ScalarFunctionCallExpression(andFn);
+ // Add the first select's condition.
+ conj.getArguments().add(new MutableObject<ILogicalExpression>(firstSelect.getCondition().getValue()));
}
+
+ // Consolidate all following selects.
+ do {
+ // Add the condition nextSelect to the new list of conjuncts.
+ conj.getArguments().add(((SelectOperator) nextSelect).getCondition());
+ selectParent = nextSelect;
+ nextSelect = (AbstractLogicalOperator) nextSelect.getInputs().get(0).getValue();
+ } while (nextSelect.getOperatorTag() == LogicalOperatorTag.SELECT);
+
+ // Hook up the input of the top-most remaining op if necessary.
+ if (topMostOp.getOperatorTag() == LogicalOperatorTag.ASSIGN || topMostOp == firstSelect) {
+ topMostOp.getInputs().set(0, selectParent.getInputs().get(0));
+ }
+
+ // Prepare for next iteration.
+ nextSelect = selectParent;
+ } while (true);
+
+ // Did we consolidate any selects?
+ if (conj == null) {
+ return false;
}
- select.getCondition().setValue(conj);
- List<Mutable<ILogicalOperator>> selInptList = select.getInputs();
- selInptList.clear();
- selInptList.add(botOpRef);
- context.computeAndSetTypeEnvironmentForOperator(select);
+
+ // Set the new conjuncts.
+ firstSelect.getCondition().setValue(conj);
+ context.computeAndSetTypeEnvironmentForOperator(firstSelect);
return true;
}
-
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java
index 5c5356c..3e6ab6f 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java
@@ -19,6 +19,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -30,7 +31,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class EliminateSubplanRule implements IAlgebraicRewriteRule {
@@ -42,7 +42,6 @@
/**
* Eliminate Subplan above ETS
- *
* and Subplan that has only ops. with one input and no free vars. (could we
* modify it to consider free vars which are sources of Unnest or Assign, if
* there are no aggregates?)
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
index 5ebaad7..de2e6af 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
@@ -9,6 +9,9 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
@@ -61,12 +64,9 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
import edu.uci.ics.hyracks.algebricks.rewriter.util.PhysicalOptimizationsUtil;
public class EnforceStructuralPropertiesRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java
index 3501467..7e27b79 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java
@@ -23,6 +23,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -41,7 +42,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.OneToOneExchangePOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.ReplicatePOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamProjectPOperator;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class ExtractCommonOperatorsRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java
index 38032dd..8305bc9 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java
@@ -16,6 +16,8 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -25,12 +27,9 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
/**
* Needed only bc. current Hyrax operators require keys to be fields.
- *
*/
public class ExtractGbyExpressionsRule extends AbstractExtractExprRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java
index 99ed003..bfaff9f 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java
@@ -22,6 +22,8 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -32,9 +34,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
public class FactorRedundantGroupAndDecorVarsRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InferTypesRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InferTypesRule.java
index e1d5549..703567a 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InferTypesRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InferTypesRule.java
@@ -2,9 +2,9 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class InferTypesRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineAssignIntoAggregateRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineAssignIntoAggregateRule.java
new file mode 100644
index 0000000..ca45cbb
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineAssignIntoAggregateRule.java
@@ -0,0 +1,122 @@
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.AbstractConstVarFunVisitor;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class InlineAssignIntoAggregateRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
+ return false;
+ }
+ boolean changed = false;
+ GroupByOperator gbyOp = (GroupByOperator) op;
+ for (ILogicalPlan p : gbyOp.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ if (inlined(r)) {
+ changed = true;
+ }
+ }
+ }
+ return changed;
+ }
+
+ private boolean inlined(Mutable<ILogicalOperator> r) throws AlgebricksException {
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getValue();
+ if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+ return false;
+ }
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+ if (op2.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+ return false;
+ }
+ AggregateOperator agg = (AggregateOperator) op1;
+ AssignOperator assign = (AssignOperator) op2;
+ VarExprSubstitution ves = new VarExprSubstitution(assign.getVariables(), assign.getExpressions());
+ for (Mutable<ILogicalExpression> exprRef : agg.getExpressions()) {
+ ILogicalExpression expr = exprRef.getValue();
+ Pair<Boolean, ILogicalExpression> p = expr.accept(ves, null);
+ if (p.first == true) {
+ exprRef.setValue(p.second);
+ }
+ // AbstractLogicalExpression ale = (AbstractLogicalExpression) expr;
+ // ale.accept(ves, null);
+ }
+ List<Mutable<ILogicalOperator>> op1InpList = op1.getInputs();
+ op1InpList.clear();
+ op1InpList.add(op2.getInputs().get(0));
+ return true;
+ }
+
+ private class VarExprSubstitution extends AbstractConstVarFunVisitor<Pair<Boolean, ILogicalExpression>, Void> {
+
+ private List<LogicalVariable> variables;
+ private List<Mutable<ILogicalExpression>> expressions;
+
+ public VarExprSubstitution(List<LogicalVariable> variables, List<Mutable<ILogicalExpression>> expressions) {
+ this.variables = variables;
+ this.expressions = expressions;
+ }
+
+ @Override
+ public Pair<Boolean, ILogicalExpression> visitConstantExpression(ConstantExpression expr, Void arg) {
+ return new Pair<Boolean, ILogicalExpression>(false, expr);
+ }
+
+ @Override
+ public Pair<Boolean, ILogicalExpression> visitFunctionCallExpression(AbstractFunctionCallExpression expr,
+ Void arg) throws AlgebricksException {
+ boolean changed = false;
+ for (Mutable<ILogicalExpression> eRef : expr.getArguments()) {
+ ILogicalExpression e = eRef.getValue();
+ Pair<Boolean, ILogicalExpression> p = e.accept(this, arg);
+ if (p.first) {
+ eRef.setValue(p.second);
+ changed = true;
+ }
+ }
+ return new Pair<Boolean, ILogicalExpression>(changed, expr);
+ }
+
+ @Override
+ public Pair<Boolean, ILogicalExpression> visitVariableReferenceExpression(VariableReferenceExpression expr,
+ Void arg) {
+ LogicalVariable v = expr.getVariableReference();
+ int idx = variables.indexOf(v);
+ if (idx < 0) {
+ return new Pair<Boolean, ILogicalExpression>(false, expr);
+ } else {
+ return new Pair<Boolean, ILogicalExpression>(true, expressions.get(idx).getValue());
+ }
+
+ }
+
+ }
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java
index 0cc2b53..8a79d81 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java
@@ -21,6 +21,8 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
@@ -39,10 +41,8 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
public class InlineVariablesRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java
index 25df119..bea8817 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java
@@ -18,6 +18,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -27,7 +28,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class InsertOuterJoinRule implements IAlgebraicRewriteRule {
@@ -38,7 +38,8 @@
}
@Override
- public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
return false;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java
index 337117e..d54833e 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java
@@ -20,6 +20,8 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
@@ -29,9 +31,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamProjectPOperator;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
public class InsertProjectBeforeUnionRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java
index 0e12bcc..f1321d3 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java
@@ -5,6 +5,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
@@ -13,7 +14,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionMergeExchangePOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SortMergeExchangePOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class IntroHashPartitionMergeExchange implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java
index acff2de..d69cd14 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java
@@ -21,6 +21,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -34,7 +35,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
public class IntroJoinInsideSubplanRule extends AbstractDecorrelationRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceAggregateCombinerRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceAggregateCombinerRule.java
new file mode 100644
index 0000000..c3d935c
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceAggregateCombinerRule.java
@@ -0,0 +1,44 @@
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+
+public class IntroduceAggregateCombinerRule extends AbstractIntroduceCombinerRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (context.checkIfInDontApplySet(this, op)) {
+ return false;
+ }
+ context.addToDontApplySet(this, op);
+ if (op.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+ return false;
+ }
+ AggregateOperator aggOp = (AggregateOperator) op;
+ if (aggOp.getExecutionMode() != ExecutionMode.PARTITIONED || aggOp.getPartitioningVariable() == null) {
+ return false;
+ }
+ Map<AggregateFunctionCallExpression, SimilarAggregatesInfo> toReplaceMap = new HashMap<AggregateFunctionCallExpression, SimilarAggregatesInfo>();
+ Pair<Boolean, Mutable<ILogicalOperator>> result = tryToPushAgg(aggOp, null, toReplaceMap, context);
+ if (!result.first || result.second == null) {
+ return false;
+ }
+ replaceOriginalAggFuncs(toReplaceMap);
+ context.computeAndSetTypeEnvironmentForOperator(aggOp);
+ return true;
+ }
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceCombinerRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceCombinerRule.java
deleted file mode 100644
index c05572b..0000000
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceCombinerRule.java
+++ /dev/null
@@ -1,323 +0,0 @@
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
-
-public class IntroduceCombinerRule implements IAlgebraicRewriteRule {
-
- @Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
- return false;
- }
-
- @Override
- public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
- throws AlgebricksException {
- AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
- if (context.checkIfInDontApplySet(this, op)) {
- return false;
- }
- context.addToDontApplySet(this, op);
- if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
- return false;
- }
- GroupByOperator gbyOp = (GroupByOperator) op;
- if (gbyOp.getExecutionMode() != ExecutionMode.PARTITIONED) {
- return false;
- }
-
- Map<AggregateFunctionCallExpression, SimilarAggregatesInfo> toReplaceMap = new HashMap<AggregateFunctionCallExpression, SimilarAggregatesInfo>();
- BookkeepingInfo bi = new BookkeepingInfo();
- bi.toReplaceMap = toReplaceMap;
- bi.modifGbyMap = new HashMap<GroupByOperator, List<LogicalVariable>>();
-
- GroupByOperator newGbyOp = opToPush(gbyOp, bi, context);
- if (newGbyOp == null) {
- return false;
- }
-
- for (Map.Entry<AggregateFunctionCallExpression, SimilarAggregatesInfo> entry : toReplaceMap.entrySet()) {
- SimilarAggregatesInfo sai = entry.getValue();
- for (AggregateExprInfo aei : sai.simAggs) {
- AbstractFunctionCallExpression afce = (AbstractFunctionCallExpression) aei.aggExprRef.getValue();
- afce.setFunctionInfo(aei.newFunInfo);
- afce.getArguments().clear();
- afce.getArguments().add(new MutableObject<ILogicalExpression>(sai.stepOneResult));
- }
- }
-
- for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyOp.getDecorList()) {
- LogicalVariable newDecorVar = context.newVar();
- newGbyOp.addDecorExpression(newDecorVar, p.second.getValue());
- p.second.setValue(new VariableReferenceExpression(newDecorVar));
- }
- newGbyOp.setExecutionMode(ExecutionMode.LOCAL);
- Object v = gbyOp.getAnnotations().get(OperatorAnnotations.USE_HASH_GROUP_BY);
- newGbyOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, v);
-
- Object v2 = gbyOp.getAnnotations().get(OperatorAnnotations.USE_EXTERNAL_GROUP_BY);
- newGbyOp.getAnnotations().put(OperatorAnnotations.USE_EXTERNAL_GROUP_BY, v2);
-
- List<LogicalVariable> propagatedVars = new LinkedList<LogicalVariable>();
- VariableUtilities.getProducedVariables(newGbyOp, propagatedVars);
-
- Set<LogicalVariable> freeVars = new HashSet<LogicalVariable>();
- OperatorPropertiesUtil.getFreeVariablesInSubplans(gbyOp, freeVars);
-
- for (LogicalVariable var : freeVars) {
- if (!propagatedVars.contains(var)) {
- LogicalVariable newDecorVar = context.newVar();
- newGbyOp.addDecorExpression(newDecorVar, new VariableReferenceExpression(var));
- VariableUtilities.substituteVariables(gbyOp.getNestedPlans().get(0).getRoots().get(0).getValue(), var,
- newDecorVar, context);
- }
- }
-
- Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0);
- opRef3.setValue(newGbyOp);
- typeGby(newGbyOp, context);
- typeGby(gbyOp, context);
- return true;
- }
-
- private void typeGby(AbstractOperatorWithNestedPlans op, IOptimizationContext context) throws AlgebricksException {
- for (ILogicalPlan p : op.getNestedPlans()) {
- OperatorPropertiesUtil.typePlan(p, context);
- }
- context.computeAndSetTypeEnvironmentForOperator(op);
- }
-
- private GroupByOperator opToPush(GroupByOperator gbyOp, BookkeepingInfo bi, IOptimizationContext context)
- throws AlgebricksException {
-
- Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0);
- ILogicalOperator op3 = opRef3.getValue();
- GroupByOperator newGbyOp = new GroupByOperator();
- newGbyOp.getInputs().add(new MutableObject<ILogicalOperator>(op3));
- // copy annotations
- Map<String, Object> annotations = newGbyOp.getAnnotations();
- for (Entry<String, Object> a : gbyOp.getAnnotations().entrySet())
- annotations.put(a.getKey(), a.getValue());
-
- List<LogicalVariable> gbyVars = gbyOp.getGbyVarList();
-
- for (ILogicalPlan p : gbyOp.getNestedPlans()) {
- Pair<Boolean, ILogicalPlan> bip = tryToPushSubplan(p, gbyOp, newGbyOp, bi, gbyVars, context);
- if (!bip.first) {
- // for now, if we cannot push everything, give up
- return null;
- }
- ILogicalPlan pushedSubplan = bip.second;
- if (pushedSubplan != null) {
- newGbyOp.getNestedPlans().add(pushedSubplan);
- }
- }
-
- ArrayList<LogicalVariable> newOpGbyList = new ArrayList<LogicalVariable>();
- ArrayList<LogicalVariable> replGbyList = new ArrayList<LogicalVariable>();
- // find maximal sequence of variable
- for (Map.Entry<GroupByOperator, List<LogicalVariable>> e : bi.modifGbyMap.entrySet()) {
- List<LogicalVariable> varList = e.getValue();
- boolean see1 = true;
- int sz1 = newOpGbyList.size();
- int i = 0;
- for (LogicalVariable v : varList) {
- if (see1) {
- if (i < sz1) {
- LogicalVariable v2 = newOpGbyList.get(i);
- if (v != v2) {
- // cannot linearize
- return null;
- }
- } else {
- see1 = false;
- newOpGbyList.add(v);
- replGbyList.add(context.newVar());
- }
- i++;
- } else {
- newOpGbyList.add(v);
- replGbyList.add(context.newVar());
- }
- }
- }
- // set the vars in the new op
- int n = newOpGbyList.size();
- for (int i = 0; i < n; i++) {
- newGbyOp.addGbyExpression(replGbyList.get(i), new VariableReferenceExpression(newOpGbyList.get(i)));
- VariableUtilities.substituteVariables(gbyOp, newOpGbyList.get(i), replGbyList.get(i), false, context);
- }
- return newGbyOp;
- }
-
- private Pair<Boolean, ILogicalPlan> tryToPushSubplan(ILogicalPlan p, GroupByOperator oldGbyOp,
- GroupByOperator newGbyOp, BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context) {
- List<Mutable<ILogicalOperator>> pushedRoots = new ArrayList<Mutable<ILogicalOperator>>();
- List<Mutable<ILogicalOperator>> toPushR = new ArrayList<Mutable<ILogicalOperator>>();
- for (Mutable<ILogicalOperator> r : p.getRoots()) {
- if (!tryToPushRoot(r, oldGbyOp, newGbyOp, bi, gbyVars, context, toPushR)) {
- // for now, if we cannot push everything, give up
- return new Pair<Boolean, ILogicalPlan>(false, null);
- }
- }
- for (Mutable<ILogicalOperator> root : toPushR) {
- pushedRoots.add(root);
- }
- if (pushedRoots.isEmpty()) {
- return new Pair<Boolean, ILogicalPlan>(true, null);
- } else {
- return new Pair<Boolean, ILogicalPlan>(true, new ALogicalPlanImpl(pushedRoots));
- }
- }
-
- private boolean tryToPushRoot(Mutable<ILogicalOperator> r, GroupByOperator oldGbyOp, GroupByOperator newGbyOp,
- BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context,
- List<Mutable<ILogicalOperator>> toPushAccumulate) {
- AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getValue();
- if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
- return false;
- }
- AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
- if (op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
- AggregateOperator initAgg = (AggregateOperator) op1;
- Pair<Boolean, Mutable<ILogicalOperator>> pOpRef = tryToPushAgg(initAgg, newGbyOp, bi.toReplaceMap, context);
- if (!pOpRef.first) {
- return false;
- }
- Mutable<ILogicalOperator> opRef = pOpRef.second;
- if (opRef != null) {
- toPushAccumulate.add(opRef);
- }
- bi.modifGbyMap.put(oldGbyOp, gbyVars);
- return true;
- } else {
- while (op2.getOperatorTag() != LogicalOperatorTag.GROUP && op2.getInputs().size() == 1) {
- op2 = (AbstractLogicalOperator) op2.getInputs().get(0).getValue();
- }
- if (op2.getOperatorTag() != LogicalOperatorTag.GROUP) {
- return false;
- }
- GroupByOperator nestedGby = (GroupByOperator) op2;
- List<LogicalVariable> gbyVars2 = nestedGby.getGbyVarList();
- List<LogicalVariable> concatGbyVars = new ArrayList<LogicalVariable>(gbyVars);
- concatGbyVars.addAll(gbyVars2);
- for (ILogicalPlan p : nestedGby.getNestedPlans()) {
- for (Mutable<ILogicalOperator> r2 : p.getRoots()) {
- if (!tryToPushRoot(r2, nestedGby, newGbyOp, bi, concatGbyVars, context, toPushAccumulate)) {
- return false;
- }
- }
- }
- return true;
- }
- }
-
- private Pair<Boolean, Mutable<ILogicalOperator>> tryToPushAgg(AggregateOperator initAgg, GroupByOperator newGbyOp,
- Map<AggregateFunctionCallExpression, SimilarAggregatesInfo> toReplaceMap, IOptimizationContext context) {
-
- ArrayList<LogicalVariable> pushedVars = new ArrayList<LogicalVariable>();
- ArrayList<Mutable<ILogicalExpression>> pushedExprs = new ArrayList<Mutable<ILogicalExpression>>();
-
- List<LogicalVariable> initVars = initAgg.getVariables();
- List<Mutable<ILogicalExpression>> initExprs = initAgg.getExpressions();
- int sz = initVars.size();
- for (int i = 0; i < sz; i++) {
- AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) initExprs.get(i).getValue();
- if (!aggFun.isTwoStep()) {
- return new Pair<Boolean, Mutable<ILogicalOperator>>(false, null);
- }
- }
-
- boolean haveAggToReplace = false;
- for (int i = 0; i < sz; i++) {
- Mutable<ILogicalExpression> expRef = initExprs.get(i);
- AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) expRef.getValue();
- IFunctionInfo fi1 = aggFun.getStepOneAggregate();
- List<Mutable<ILogicalExpression>> newArgs = new ArrayList<Mutable<ILogicalExpression>>(aggFun
- .getArguments().size());
- for (Mutable<ILogicalExpression> er : aggFun.getArguments()) {
- newArgs.add(new MutableObject<ILogicalExpression>(er.getValue().cloneExpression()));
- }
- // AggregateFunctionCallExpression aggLocal = new AggregateFunctionCallExpression(fi1, false, newArgs);
- // pushedExprs.add(new Mutable<ILogicalExpression>(aggLocal));
-
- IFunctionInfo fi2 = aggFun.getStepTwoAggregate();
-
- SimilarAggregatesInfo inf = toReplaceMap.get(aggFun);
- if (inf == null) {
- inf = new SimilarAggregatesInfo();
- LogicalVariable newAggVar = context.newVar();
- pushedVars.add(newAggVar);
- inf.stepOneResult = new VariableReferenceExpression(newAggVar);
- inf.simAggs = new ArrayList<AggregateExprInfo>();
- toReplaceMap.put(aggFun, inf);
- AggregateFunctionCallExpression aggLocal = new AggregateFunctionCallExpression(fi1, false, newArgs);
- pushedExprs.add(new MutableObject<ILogicalExpression>(aggLocal));
- }
- AggregateExprInfo aei = new AggregateExprInfo();
- aei.aggExprRef = expRef;
- aei.newFunInfo = fi2;
- inf.simAggs.add(aei);
- haveAggToReplace = true;
- }
-
- if (!pushedVars.isEmpty()) {
- AggregateOperator pushedAgg = new AggregateOperator(pushedVars, pushedExprs);
- pushedAgg.setExecutionMode(ExecutionMode.LOCAL);
- NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(newGbyOp));
- nts.setExecutionMode(ExecutionMode.LOCAL);
- pushedAgg.getInputs().add(new MutableObject<ILogicalOperator>(nts));
- return new Pair<Boolean, Mutable<ILogicalOperator>>(true, new MutableObject<ILogicalOperator>(pushedAgg));
- } else {
- return new Pair<Boolean, Mutable<ILogicalOperator>>(haveAggToReplace, null);
- }
- }
-
- private class SimilarAggregatesInfo {
- ILogicalExpression stepOneResult;
- List<AggregateExprInfo> simAggs;
- }
-
- private class AggregateExprInfo {
- Mutable<ILogicalExpression> aggExprRef;
- IFunctionInfo newFunInfo;
- }
-
- private class BookkeepingInfo {
- Map<AggregateFunctionCallExpression, SimilarAggregatesInfo> toReplaceMap;
- Map<GroupByOperator, List<LogicalVariable>> modifGbyMap;
- }
-}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByCombinerRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByCombinerRule.java
new file mode 100644
index 0000000..5c5fdb1
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByCombinerRule.java
@@ -0,0 +1,218 @@
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+
+public class IntroduceGroupByCombinerRule extends AbstractIntroduceCombinerRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (context.checkIfInDontApplySet(this, op)) {
+ return false;
+ }
+ context.addToDontApplySet(this, op);
+ if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
+ return false;
+ }
+ GroupByOperator gbyOp = (GroupByOperator) op;
+ if (gbyOp.getExecutionMode() != ExecutionMode.PARTITIONED) {
+ return false;
+ }
+
+ BookkeepingInfo bi = new BookkeepingInfo();
+ GroupByOperator newGbyOp = opToPush(gbyOp, bi, context);
+ if (newGbyOp == null) {
+ return false;
+ }
+
+ replaceOriginalAggFuncs(bi.toReplaceMap);
+
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyOp.getDecorList()) {
+ LogicalVariable newDecorVar = context.newVar();
+ newGbyOp.addDecorExpression(newDecorVar, p.second.getValue());
+ p.second.setValue(new VariableReferenceExpression(newDecorVar));
+ }
+ newGbyOp.setExecutionMode(ExecutionMode.LOCAL);
+ Object v = gbyOp.getAnnotations().get(OperatorAnnotations.USE_HASH_GROUP_BY);
+ newGbyOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, v);
+
+ Object v2 = gbyOp.getAnnotations().get(OperatorAnnotations.USE_EXTERNAL_GROUP_BY);
+ newGbyOp.getAnnotations().put(OperatorAnnotations.USE_EXTERNAL_GROUP_BY, v2);
+
+ List<LogicalVariable> propagatedVars = new LinkedList<LogicalVariable>();
+ VariableUtilities.getProducedVariables(newGbyOp, propagatedVars);
+
+ Set<LogicalVariable> freeVars = new HashSet<LogicalVariable>();
+ OperatorPropertiesUtil.getFreeVariablesInSubplans(gbyOp, freeVars);
+
+ for (LogicalVariable var : freeVars) {
+ if (!propagatedVars.contains(var)) {
+ LogicalVariable newDecorVar = context.newVar();
+ newGbyOp.addDecorExpression(newDecorVar, new VariableReferenceExpression(var));
+ VariableUtilities.substituteVariables(gbyOp.getNestedPlans().get(0).getRoots().get(0).getValue(), var,
+ newDecorVar, context);
+ }
+ }
+
+ Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0);
+ opRef3.setValue(newGbyOp);
+ typeGby(newGbyOp, context);
+ typeGby(gbyOp, context);
+ return true;
+ }
+
+ private void typeGby(AbstractOperatorWithNestedPlans op, IOptimizationContext context) throws AlgebricksException {
+ for (ILogicalPlan p : op.getNestedPlans()) {
+ OperatorPropertiesUtil.typePlan(p, context);
+ }
+ context.computeAndSetTypeEnvironmentForOperator(op);
+ }
+
+ private GroupByOperator opToPush(GroupByOperator gbyOp, BookkeepingInfo bi, IOptimizationContext context)
+ throws AlgebricksException {
+ // Hook up input to new group-by.
+ Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0);
+ ILogicalOperator op3 = opRef3.getValue();
+ GroupByOperator newGbyOp = new GroupByOperator();
+ newGbyOp.getInputs().add(new MutableObject<ILogicalOperator>(op3));
+ // Copy annotations.
+ Map<String, Object> annotations = newGbyOp.getAnnotations();
+ annotations.putAll(gbyOp.getAnnotations());
+
+ List<LogicalVariable> gbyVars = gbyOp.getGbyVarList();
+ for (ILogicalPlan p : gbyOp.getNestedPlans()) {
+ Pair<Boolean, ILogicalPlan> bip = tryToPushSubplan(p, gbyOp, newGbyOp, bi, gbyVars, context);
+ if (!bip.first) {
+ // For now, if we cannot push everything, give up.
+ return null;
+ }
+ ILogicalPlan pushedSubplan = bip.second;
+ if (pushedSubplan != null) {
+ newGbyOp.getNestedPlans().add(pushedSubplan);
+ }
+ }
+
+ ArrayList<LogicalVariable> newOpGbyList = new ArrayList<LogicalVariable>();
+ ArrayList<LogicalVariable> replGbyList = new ArrayList<LogicalVariable>();
+ // Find maximal sequence of variable.
+ for (Map.Entry<GroupByOperator, List<LogicalVariable>> e : bi.modifyGbyMap.entrySet()) {
+ List<LogicalVariable> varList = e.getValue();
+ boolean see1 = true;
+ int sz1 = newOpGbyList.size();
+ int i = 0;
+ for (LogicalVariable v : varList) {
+ if (see1) {
+ if (i < sz1) {
+ LogicalVariable v2 = newOpGbyList.get(i);
+ if (v != v2) {
+ // cannot linearize
+ return null;
+ }
+ } else {
+ see1 = false;
+ newOpGbyList.add(v);
+ replGbyList.add(context.newVar());
+ }
+ i++;
+ } else {
+ newOpGbyList.add(v);
+ replGbyList.add(context.newVar());
+ }
+ }
+ }
+ // set the vars in the new op
+ int n = newOpGbyList.size();
+ for (int i = 0; i < n; i++) {
+ newGbyOp.addGbyExpression(replGbyList.get(i), new VariableReferenceExpression(newOpGbyList.get(i)));
+ VariableUtilities.substituteVariables(gbyOp, newOpGbyList.get(i), replGbyList.get(i), false, context);
+ }
+ return newGbyOp;
+ }
+
+ private Pair<Boolean, ILogicalPlan> tryToPushSubplan(ILogicalPlan nestedPlan, GroupByOperator oldGbyOp,
+ GroupByOperator newGbyOp, BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context)
+ throws AlgebricksException {
+ List<Mutable<ILogicalOperator>> pushedRoots = new ArrayList<Mutable<ILogicalOperator>>();
+ for (Mutable<ILogicalOperator> r : nestedPlan.getRoots()) {
+ if (!tryToPushRoot(r, oldGbyOp, newGbyOp, bi, gbyVars, context, pushedRoots)) {
+ // For now, if we cannot push everything, give up.
+ return new Pair<Boolean, ILogicalPlan>(false, null);
+ }
+ }
+ if (pushedRoots.isEmpty()) {
+ return new Pair<Boolean, ILogicalPlan>(true, null);
+ } else {
+ return new Pair<Boolean, ILogicalPlan>(true, new ALogicalPlanImpl(pushedRoots));
+ }
+ }
+
+ private boolean tryToPushRoot(Mutable<ILogicalOperator> root, GroupByOperator oldGbyOp, GroupByOperator newGbyOp,
+ BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context,
+ List<Mutable<ILogicalOperator>> toPushAccumulate) throws AlgebricksException {
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) root.getValue();
+ if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+ return false;
+ }
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+ if (op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+ AggregateOperator initAgg = (AggregateOperator) op1;
+ Pair<Boolean, Mutable<ILogicalOperator>> pOpRef = tryToPushAgg(initAgg, newGbyOp, bi.toReplaceMap, context);
+ if (!pOpRef.first) {
+ return false;
+ }
+ Mutable<ILogicalOperator> opRef = pOpRef.second;
+ if (opRef != null) {
+ toPushAccumulate.add(opRef);
+ }
+ bi.modifyGbyMap.put(oldGbyOp, gbyVars);
+ return true;
+ } else {
+ while (op2.getOperatorTag() != LogicalOperatorTag.GROUP && op2.getInputs().size() == 1) {
+ op2 = (AbstractLogicalOperator) op2.getInputs().get(0).getValue();
+ }
+ if (op2.getOperatorTag() != LogicalOperatorTag.GROUP) {
+ return false;
+ }
+ GroupByOperator nestedGby = (GroupByOperator) op2;
+ List<LogicalVariable> gbyVars2 = nestedGby.getGbyVarList();
+ List<LogicalVariable> concatGbyVars = new ArrayList<LogicalVariable>(gbyVars);
+ concatGbyVars.addAll(gbyVars2);
+ for (ILogicalPlan p : nestedGby.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r2 : p.getRoots()) {
+ if (!tryToPushRoot(r2, nestedGby, newGbyOp, bi, concatGbyVars, context, toPushAccumulate)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+ }
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForStandaloneAggregRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForStandaloneAggregRule.java
deleted file mode 100644
index 34060c2..0000000
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForStandaloneAggregRule.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
-
-/**
- *
- * When aggregates appear w/o group-by, a default group by a constant is
- * introduced.
- *
- */
-
-public class IntroduceGroupByForStandaloneAggregRule implements IAlgebraicRewriteRule {
-
- @Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
- return false;
- }
-
- @Override
- public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
- throws AlgebricksException {
- AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
- if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
- return false;
- }
- Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
- AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
- if (op2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
- return false;
- }
-
- AssignOperator assign = (AssignOperator) op;
- AggregateOperator agg = (AggregateOperator) op2;
- if (agg.getVariables().size() != 1) {
- return false;
- }
- LogicalVariable aggVar = agg.getVariables().get(0);
- List<LogicalVariable> used = new LinkedList<LogicalVariable>();
- VariableUtilities.getUsedVariables(assign, used);
- if (used.contains(aggVar)) {
- Mutable<ILogicalOperator> opRef3 = op2.getInputs().get(0);
- List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
- LogicalVariable gbyVar = context.newVar();
- // ILogicalExpression constOne = new ConstantExpression(new
- // IntegerLiteral(new Integer(1)));
- groupByList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(gbyVar,
- new MutableObject<ILogicalExpression>(ConstantExpression.TRUE)));
- NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>());
- List<Mutable<ILogicalOperator>> aggInpList = agg.getInputs();
- aggInpList.clear();
- aggInpList.add(new MutableObject<ILogicalOperator>(nts));
- ILogicalPlan np1 = new ALogicalPlanImpl(opRef2);
- ArrayList<ILogicalPlan> nestedPlans = new ArrayList<ILogicalPlan>();
- nestedPlans.add(np1);
- GroupByOperator gbyOp = new GroupByOperator(groupByList,
- new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>(), nestedPlans);
- Mutable<ILogicalOperator> opRefGby = new MutableObject<ILogicalOperator>(gbyOp);
- nts.getDataSourceReference().setValue(gbyOp);
- gbyOp.getInputs().add(opRef3);
- List<Mutable<ILogicalOperator>> asgnInpList = assign.getInputs();
- context.computeAndSetTypeEnvironmentForOperator(nts);
- context.computeAndSetTypeEnvironmentForOperator(agg);
- context.computeAndSetTypeEnvironmentForOperator(gbyOp);
- asgnInpList.clear();
- asgnInpList.add(opRefGby);
- return true;
- }
- return false;
- }
-
-}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java
index 87297e7..18c483e 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java
@@ -26,6 +26,8 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -53,10 +55,8 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
import edu.uci.ics.hyracks.algebricks.rewriter.util.PhysicalOptimizationsUtil;
public class IntroduceGroupByForSubplanRule implements IAlgebraicRewriteRule {
@@ -176,10 +176,10 @@
context.computeAndSetTypeEnvironmentForOperator(tmpAsgn);
}
- IFunctionInfo finfoEq = AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.IS_NULL);
+ IFunctionInfo finfoEq = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.IS_NULL);
ILogicalExpression isNullTest = new ScalarFunctionCallExpression(finfoEq,
new MutableObject<ILogicalExpression>(new VariableReferenceExpression(testForNull)));
- IFunctionInfo finfoNot = AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.NOT);
+ IFunctionInfo finfoNot = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.NOT);
ScalarFunctionCallExpression nonNullTest = new ScalarFunctionCallExpression(finfoNot,
new MutableObject<ILogicalExpression>(isNullTest));
SelectOperator selectNonNull = new SelectOperator(new MutableObject<ILogicalExpression>(nonNullTest));
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
index 194b818..ae57fb7 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
@@ -17,6 +17,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
@@ -27,7 +28,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.OneToOneExchangePOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class IsolateHyracksOperatorsRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
index 7ee1f49..75862cf 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
@@ -20,6 +20,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -33,7 +34,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class PullSelectOutOfEqJoin implements IAlgebraicRewriteRule {
@@ -63,7 +63,7 @@
}
AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) expr;
FunctionIdentifier fi = fexp.getFunctionIdentifier();
- if (fi != AlgebricksBuiltinFunctions.AND) {
+ if (!fi.equals(AlgebricksBuiltinFunctions.AND)) {
return false;
}
eqVarVarComps.clear();
@@ -79,9 +79,9 @@
return false;
}
// pull up
- ILogicalExpression pulledCond = makeCondition(otherPredicates);
+ ILogicalExpression pulledCond = makeCondition(otherPredicates, context);
SelectOperator select = new SelectOperator(new MutableObject<ILogicalExpression>(pulledCond));
- ILogicalExpression newJoinCond = makeCondition(eqVarVarComps);
+ ILogicalExpression newJoinCond = makeCondition(eqVarVarComps, context);
join.getCondition().setValue(newJoinCond);
select.getInputs().add(new MutableObject<ILogicalOperator>(join));
opRef.setValue(select);
@@ -89,9 +89,9 @@
return true;
}
- private ILogicalExpression makeCondition(List<Mutable<ILogicalExpression>> predList) {
+ private ILogicalExpression makeCondition(List<Mutable<ILogicalExpression>> predList, IOptimizationContext context) {
if (predList.size() > 1) {
- IFunctionInfo finfo = AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.AND);
+ IFunctionInfo finfo = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.AND);
return new ScalarFunctionCallExpression(finfo, predList);
} else {
return predList.get(0).getValue();
@@ -103,7 +103,7 @@
return false;
}
AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr;
- if (f.getFunctionIdentifier() != AlgebricksBuiltinFunctions.EQ) {
+ if (!f.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.EQ)) {
return false;
}
ILogicalExpression e1 = f.getArguments().get(0).getValue();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java
index 5eb7160..c105b51 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java
@@ -19,6 +19,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
@@ -27,7 +28,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class PushAssignDownThroughProductRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushDieUpRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushDieUpRule.java
index ef7822f..c4dd78d 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushDieUpRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushDieUpRule.java
@@ -3,11 +3,11 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class PushDieUpRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushLimitDownRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushLimitDownRule.java
index 1b096ea..acfdbd3 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushLimitDownRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushLimitDownRule.java
@@ -19,6 +19,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
@@ -32,7 +33,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamLimitPOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class PushLimitDownRule implements IAlgebraicRewriteRule {
@@ -45,7 +45,6 @@
/**
* When a global Limit over a merge-exchange is found, a local Limit is
* pushed down.
- *
*/
@Override
@@ -101,8 +100,8 @@
clone2 = new LimitOperator(opLim.getMaxObjects().getValue(), false);
} else {
// push limit (max+offset)
- IFunctionInfo finfoAdd = AlgebricksBuiltinFunctions
- .getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.NUMERIC_ADD);
+ IFunctionInfo finfoAdd = context.getMetadataProvider().lookupFunction(
+ AlgebricksBuiltinFunctions.NUMERIC_ADD);
ScalarFunctionCallExpression maxPlusOffset = new ScalarFunctionCallExpression(finfoAdd,
opLim.getMaxObjects(), opLim.getOffset());
clone2 = new LimitOperator(maxPlusOffset, false);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java
index b925cfe..95ad030 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java
@@ -19,6 +19,8 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -34,9 +36,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractPhysicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StableSortPOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
public class PushNestedOrderByUnderPreSortedGroupByRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java
index b70d88d..f223844 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java
@@ -23,6 +23,8 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -34,17 +36,13 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
/**
- *
* Pushes projections through its input operator, provided that operator does
* not produce the projected variables.
*
* @author Nicola
- *
*/
public class PushProjectDownRule implements IAlgebraicRewriteRule {
@@ -84,7 +82,8 @@
if (op2.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE
|| op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE
|| op2.getOperatorTag() == LogicalOperatorTag.PROJECT
- || op2.getOperatorTag() == LogicalOperatorTag.REPLICATE) {
+ || op2.getOperatorTag() == LogicalOperatorTag.REPLICATE
+ || op2.getOperatorTag() == LogicalOperatorTag.UNIONALL) {
return new Pair<Boolean, Boolean>(false, false);
}
if (!op2.isMap()) {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java
index 4e14f9a..2cbbdbf 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java
@@ -2,13 +2,13 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class PushProjectIntoDataSourceScanRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java
index fedcad9..ef4fd2e 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java
@@ -19,6 +19,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
@@ -27,7 +28,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class PushSelectDownRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java
index 315abdb..8c679c5 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java
@@ -25,6 +25,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -39,7 +40,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class PushSelectIntoJoinRule implements IAlgebraicRewriteRule {
@@ -148,7 +148,7 @@
pushOps(pushedOnLeft, joinBranchLeftRef, context);
pushOps(pushedOnRight, joinBranchRightRef, context);
}
- addCondToJoin(select, join);
+ addCondToJoin(select, join, context);
} else { // push down
Iterator<Mutable<ILogicalOperator>> branchIter = join.getInputs().iterator();
@@ -202,7 +202,8 @@
joinBranch.setValue(topOp);
}
- private static void addCondToJoin(SelectOperator select, AbstractBinaryJoinOperator join) {
+ private static void addCondToJoin(SelectOperator select, AbstractBinaryJoinOperator join,
+ IOptimizationContext context) {
ILogicalExpression cond = join.getCondition().getValue();
if (OperatorPropertiesUtil.isAlwaysTrueCond(cond)) { // the join was a product
join.getCondition().setValue(select.getCondition().getValue());
@@ -211,8 +212,8 @@
if (cond.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
AbstractFunctionCallExpression fcond = (AbstractFunctionCallExpression) cond;
if (fcond.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.AND)) {
- AbstractFunctionCallExpression newCond = new ScalarFunctionCallExpression(
- AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.AND));
+ AbstractFunctionCallExpression newCond = new ScalarFunctionCallExpression(context
+ .getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.AND));
newCond.getArguments().add(select.getCondition());
newCond.getArguments().addAll(fcond.getArguments());
join.getCondition().setValue(newCond);
@@ -220,9 +221,9 @@
}
}
if (!bAddedToConj) {
- AbstractFunctionCallExpression newCond = new ScalarFunctionCallExpression(
- AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.AND),
- select.getCondition(), new MutableObject<ILogicalExpression>(join.getCondition().getValue()));
+ AbstractFunctionCallExpression newCond = new ScalarFunctionCallExpression(context.getMetadataProvider()
+ .lookupFunction(AlgebricksBuiltinFunctions.AND), select.getCondition(),
+ new MutableObject<ILogicalExpression>(join.getCondition().getValue()));
join.getCondition().setValue(newCond);
}
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java
index 25855e5..d7bd543 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java
@@ -21,6 +21,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -31,7 +32,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class PushSubplanWithAggregateDownThroughProductRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java
index f06a2ac..e18a380 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java
@@ -2,12 +2,12 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class ReinferAllTypesRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java
index cdc6086..f5ebe77 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java
@@ -19,6 +19,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
@@ -26,7 +27,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
/*
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
index ff1a102..c53ea0a 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
@@ -22,6 +22,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -32,8 +33,8 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class RemoveUnusedAssignAndAggregateRule implements IAlgebraicRewriteRule {
@@ -98,6 +99,12 @@
context.computeAndSetTypeEnvironmentForOperator(agg);
}
return agg.getVariables().size();
+ } else if (op.getOperatorTag() == LogicalOperatorTag.UNNEST) {
+ UnnestOperator uOp = (UnnestOperator) op;
+ LogicalVariable pVar = uOp.getPositionalVariable();
+ if (pVar != null && toRemove.contains(pVar)) {
+ uOp.setPositionalVariable(null);
+ }
}
return -1;
}
@@ -141,6 +148,12 @@
} else if (op.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
AggregateOperator agg = (AggregateOperator) op;
toRemove.addAll(agg.getVariables());
+ } else if (op.getOperatorTag() == LogicalOperatorTag.UNNEST) {
+ UnnestOperator uOp = (UnnestOperator) op;
+ LogicalVariable pVar = uOp.getPositionalVariable();
+ if (pVar != null) {
+ toRemove.add(pVar);
+ }
}
List<LogicalVariable> used = new LinkedList<LogicalVariable>();
VariableUtilities.getUsedVariables(op, used);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
index f1f3533..38cf96e 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
@@ -6,6 +6,9 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
@@ -58,11 +61,8 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.UnionAllPOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.UnnestPOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.WriteResultPOperator;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
import edu.uci.ics.hyracks.algebricks.rewriter.util.JoinUtils;
public class SetAlgebricksPhysicalOperatorsRule implements IAlgebraicRewriteRule {
@@ -262,13 +262,13 @@
break;
}
case INDEX_INSERT_DELETE: {
- IndexInsertDeleteOperator opLoad = (IndexInsertDeleteOperator) op;
+ IndexInsertDeleteOperator opInsDel = (IndexInsertDeleteOperator) op;
List<LogicalVariable> primaryKeys = new ArrayList<LogicalVariable>();
List<LogicalVariable> secondaryKeys = new ArrayList<LogicalVariable>();
- getKeys(opLoad.getPrimaryKeyExpressions(), primaryKeys);
- getKeys(opLoad.getSecondaryKeyExpressions(), secondaryKeys);
- op.setPhysicalOperator(new IndexInsertDeletePOperator(primaryKeys, secondaryKeys, opLoad
- .getDataSourceIndex()));
+ getKeys(opInsDel.getPrimaryKeyExpressions(), primaryKeys);
+ getKeys(opInsDel.getSecondaryKeyExpressions(), secondaryKeys);
+ op.setPhysicalOperator(new IndexInsertDeletePOperator(primaryKeys, secondaryKeys,
+ opInsDel.getFilterExpression(), opInsDel.getDataSourceIndex()));
break;
}
case SINK: {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java
index bdd56b6..5bc3432 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java
@@ -16,23 +16,20 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
/**
- *
* This rule sets the executionMode property of an operator, w/o introducing
* EXCHANGE operators in the plan. Previously, i.e. before having physical
* optimizations in place, we were using the IntroduceExchangeRule, which was
* doing both, to both set excutionMode and introduce data exchange ops.
*
- *
* @author Nicola
- *
*/
public class SetExecutionModeRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java
index fb7a49f..b79c31b 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java
@@ -14,21 +14,26 @@
*/
package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class SimpleUnnestToProductRule implements IAlgebraicRewriteRule {
@@ -42,7 +47,8 @@
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
throws AlgebricksException {
AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
- if (op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) {
+ if (op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN
+ && op.getOperatorTag() != LogicalOperatorTag.UNNEST) {
return false;
}
@@ -52,6 +58,11 @@
if (!(op2 instanceof AbstractScanOperator) && !descOrSelfIsSourceScan(op2)) {
return false;
}
+ // Make sure that op does not use any variables produced by op2.
+ if (!opsAreIndependent(op, op2)) {
+ return false;
+ }
+
InnerJoinOperator product = new InnerJoinOperator(
new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
@@ -69,7 +80,12 @@
}
private boolean descOrSelfIsSourceScan(AbstractLogicalOperator op2) {
- if (op2.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
+ // Disregard data source scans in a subplan.
+ if (op2.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
+ return false;
+ }
+ if (op2.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN
+ && op2.getOperatorTag() != LogicalOperatorTag.UNNEST) {
return true;
}
for (Mutable<ILogicalOperator> cRef : op2.getInputs()) {
@@ -81,4 +97,17 @@
return false;
}
+ private boolean opsAreIndependent(ILogicalOperator unnestOp, ILogicalOperator outer) throws AlgebricksException {
+ List<LogicalVariable> opUsedVars = new ArrayList<LogicalVariable>();
+ VariableUtilities.getUsedVariables(unnestOp, opUsedVars);
+ Set<LogicalVariable> op2LiveVars = new HashSet<LogicalVariable>();
+ VariableUtilities.getLiveVariables(outer, op2LiveVars);
+ for (LogicalVariable usedVar : opUsedVars) {
+ if (op2LiveVars.contains(usedVar)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java
index 7c31fb9..09e03b5 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java
@@ -21,6 +21,7 @@
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -31,26 +32,16 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
/**
- *
* Looks for a nested group-by plan ending in
- *
* subplan {
- *
* ...
- *
* }
- *
* select (function-call: algebricks:not, Args:[function-call:
* algebricks:is-null, Args:[...]])
- *
* nested tuple source -- |UNPARTITIONED|
- *
- *
- *
*/
public class SubplanOutOfGroupRule implements IAlgebraicRewriteRule {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/JoinUtils.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/JoinUtils.java
index 1714a24..ddc00e3 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/JoinUtils.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/JoinUtils.java
@@ -21,6 +21,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -41,7 +42,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.InMemoryHashJoinPOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.NLJoinPOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
public class JoinUtils {
@@ -134,7 +134,7 @@
case FUNCTION_CALL: {
AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) e;
FunctionIdentifier fi = fexp.getFunctionIdentifier();
- if (fi == AlgebricksBuiltinFunctions.AND) {
+ if (fi.equals(AlgebricksBuiltinFunctions.AND)) {
for (Mutable<ILogicalExpression> a : fexp.getArguments()) {
if (!isHashJoinCondition(a.getValue(), inLeftAll, inRightAll, outLeftFields,
outRightFields)) {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java
index 795c767..d7ebfb4 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java
@@ -5,6 +5,7 @@
import org.apache.commons.lang3.mutable.Mutable;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -13,7 +14,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.FDsAndEquivClassesVisitor;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
public class PhysicalOptimizationsUtil {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/pom.xml b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/pom.xml
new file mode 100644
index 0000000..79bcaeb
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/pom.xml
@@ -0,0 +1,51 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-algebricks-runtime</artifactId>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-btree</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-rtree</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-dataflow-std</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks-common</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks-data</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/aggregators/TupleCountAggregateFunctionFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/aggregators/TupleCountAggregateFunctionFactory.java
new file mode 100644
index 0000000..0658d62
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/aggregators/TupleCountAggregateFunctionFactory.java
@@ -0,0 +1,47 @@
+package edu.uci.ics.hyracks.algebricks.runtime.aggregators;
+
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class TupleCountAggregateFunctionFactory implements IAggregateEvaluatorFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IAggregateEvaluator createAggregateEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
+ final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ return new IAggregateEvaluator() {
+
+ int cnt;
+
+ @Override
+ public void step(IFrameTupleReference tuple) throws AlgebricksException {
+ ++cnt;
+ }
+
+ @Override
+ public void init() throws AlgebricksException {
+ cnt = 0;
+ }
+
+ @Override
+ public void finish(IPointable result) throws AlgebricksException {
+ try {
+ abvs.reset();
+ abvs.getDataOutput().writeInt(cnt);
+ result.set(abvs);
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+ };
+ }
+
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/aggregators/TupleCountRunningAggregateFunctionFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/aggregators/TupleCountRunningAggregateFunctionFactory.java
new file mode 100644
index 0000000..fa73f6c
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/aggregators/TupleCountRunningAggregateFunctionFactory.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.runtime.aggregators;
+
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class TupleCountRunningAggregateFunctionFactory implements IRunningAggregateEvaluatorFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IRunningAggregateEvaluator createRunningAggregateEvaluator() throws AlgebricksException {
+ final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ return new IRunningAggregateEvaluator() {
+
+ int cnt;
+
+ @Override
+ public void step(IFrameTupleReference tuple, IPointable result) throws AlgebricksException {
+ ++cnt;
+ try {
+ abvs.reset();
+ abvs.getDataOutput().writeInt(cnt);
+ result.set(abvs);
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
+ @Override
+ public void init() throws AlgebricksException {
+ cnt = 0;
+ }
+ };
+ }
+
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/AlgebricksPipeline.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/AlgebricksPipeline.java
similarity index 95%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/AlgebricksPipeline.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/AlgebricksPipeline.java
index ecd3e96..50f014b 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/AlgebricksPipeline.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/AlgebricksPipeline.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
import java.io.Serializable;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IAggregateEvaluator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IAggregateEvaluator.java
new file mode 100644
index 0000000..9b014ee
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IAggregateEvaluator.java
@@ -0,0 +1,13 @@
+package edu.uci.ics.hyracks.algebricks.runtime.base;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public interface IAggregateEvaluator {
+ public void init() throws AlgebricksException;
+
+ public void step(IFrameTupleReference tuple) throws AlgebricksException;
+
+ public void finish(IPointable result) throws AlgebricksException;
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IAggregateEvaluatorFactory.java
similarity index 64%
copy from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
copy to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IAggregateEvaluatorFactory.java
index 871fe43..83bfbf7 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IAggregateEvaluatorFactory.java
@@ -12,10 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
import java.io.Serializable;
-public interface IPrinterFactory extends Serializable {
- public IPrinter createPrinter();
-}
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+
+public interface IAggregateEvaluatorFactory extends Serializable {
+ public IAggregateEvaluator createAggregateEvaluator(IHyracksTaskContext ctx) throws AlgebricksException;
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IAggregateFunction.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyAggregateFunction.java
similarity index 68%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IAggregateFunction.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyAggregateFunction.java
index e68ae52..c768674 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IAggregateFunction.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyAggregateFunction.java
@@ -1,9 +1,9 @@
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-public interface IAggregateFunction {
+public interface ICopyAggregateFunction {
/** should be called each time a new aggregate value is computed */
public void init() throws AlgebricksException;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyAggregateFunctionFactory.java
similarity index 63%
copy from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntimeFactory.java
copy to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyAggregateFunctionFactory.java
index 4e2d113..702f392 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyAggregateFunctionFactory.java
@@ -12,13 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
import java.io.Serializable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
-public interface IPushRuntimeFactory extends Serializable {
- public IPushRuntime createPushRuntime(RuntimeContext context) throws AlgebricksException;
+public interface ICopyAggregateFunctionFactory extends Serializable {
+ public ICopyAggregateFunction createAggregateFunction(IDataOutputProvider provider) throws AlgebricksException;
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IEvaluator.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyEvaluator.java
similarity index 81%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IEvaluator.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyEvaluator.java
index 566651d..58d7569 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IEvaluator.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyEvaluator.java
@@ -12,11 +12,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-public interface IEvaluator {
+public interface ICopyEvaluator {
public void evaluate(IFrameTupleReference tuple) throws AlgebricksException;
}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyEvaluatorFactory.java
similarity index 65%
copy from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
copy to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyEvaluatorFactory.java
index 871fe43..649902c 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyEvaluatorFactory.java
@@ -12,10 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
import java.io.Serializable;
-public interface IPrinterFactory extends Serializable {
- public IPrinter createPrinter();
-}
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
+
+public interface ICopyEvaluatorFactory extends Serializable {
+ public ICopyEvaluator createEvaluator(IDataOutputProvider output) throws AlgebricksException;
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IRunningAggregateFunction.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyRunningAggregateFunction.java
similarity index 81%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IRunningAggregateFunction.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyRunningAggregateFunction.java
index ac381cf..3b14b28 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IRunningAggregateFunction.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyRunningAggregateFunction.java
@@ -12,12 +12,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-public interface IRunningAggregateFunction {
+public interface ICopyRunningAggregateFunction {
public void init() throws AlgebricksException;
public void step(IFrameTupleReference tuple) throws AlgebricksException;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IRunningAggregateFunctionFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyRunningAggregateFunctionFactory.java
similarity index 64%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IRunningAggregateFunctionFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyRunningAggregateFunctionFactory.java
index 638f5f8..1065020 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IRunningAggregateFunctionFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyRunningAggregateFunctionFactory.java
@@ -12,14 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
import java.io.Serializable;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
-public interface IRunningAggregateFunctionFactory extends Serializable {
- public IRunningAggregateFunction createRunningAggregateFunction(IDataOutputProvider provider)
+public interface ICopyRunningAggregateFunctionFactory extends Serializable {
+ public ICopyRunningAggregateFunction createRunningAggregateFunction(IDataOutputProvider provider)
throws AlgebricksException;
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/ISerializableAggregateFunction.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopySerializableAggregateFunction.java
similarity index 84%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/ISerializableAggregateFunction.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopySerializableAggregateFunction.java
index 4e9c5fb..47aec7d 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/ISerializableAggregateFunction.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopySerializableAggregateFunction.java
@@ -1,11 +1,11 @@
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
import java.io.DataOutput;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-public interface ISerializableAggregateFunction {
+public interface ICopySerializableAggregateFunction {
/**
* initialize the space occupied by internal state
*
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopySerializableAggregateFunctionFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopySerializableAggregateFunctionFactory.java
new file mode 100644
index 0000000..69d56a8
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopySerializableAggregateFunctionFactory.java
@@ -0,0 +1,9 @@
+package edu.uci.ics.hyracks.algebricks.runtime.base;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+
+public interface ICopySerializableAggregateFunctionFactory extends Serializable {
+ public ICopySerializableAggregateFunction createAggregateFunction() throws AlgebricksException;
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunction.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyUnnestingFunction.java
similarity index 82%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunction.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyUnnestingFunction.java
index 5b891f2..3443403 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunction.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyUnnestingFunction.java
@@ -12,12 +12,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-public interface IUnnestingFunction {
+public interface ICopyUnnestingFunction {
public void init(IFrameTupleReference tuple) throws AlgebricksException;
public boolean step() throws AlgebricksException;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyUnnestingFunctionFactory.java
similarity index 63%
copy from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntimeFactory.java
copy to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyUnnestingFunctionFactory.java
index 4e2d113..e43da8b 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyUnnestingFunctionFactory.java
@@ -12,13 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
import java.io.Serializable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
-public interface IPushRuntimeFactory extends Serializable {
- public IPushRuntime createPushRuntime(RuntimeContext context) throws AlgebricksException;
+public interface ICopyUnnestingFunctionFactory extends Serializable {
+ public ICopyUnnestingFunction createUnnestingFunction(IDataOutputProvider provider) throws AlgebricksException;
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntime.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IPushRuntime.java
similarity index 93%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntime.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IPushRuntime.java
index 01ae150..26fcc67 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntime.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IPushRuntime.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IPushRuntimeFactory.java
similarity index 69%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IPushRuntimeFactory.java
index 4e2d113..f1d01e6 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IPushRuntimeFactory.java
@@ -12,13 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
import java.io.Serializable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
public interface IPushRuntimeFactory extends Serializable {
- public IPushRuntime createPushRuntime(RuntimeContext context) throws AlgebricksException;
+ public IPushRuntime createPushRuntime(IHyracksTaskContext ctx) throws AlgebricksException;
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IRunningAggregateFunction.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IRunningAggregateEvaluator.java
similarity index 71%
copy from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IRunningAggregateFunction.java
copy to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IRunningAggregateEvaluator.java
index ac381cf..f76a33c 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IRunningAggregateFunction.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IRunningAggregateEvaluator.java
@@ -12,13 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-public interface IRunningAggregateFunction {
+public interface IRunningAggregateEvaluator {
public void init() throws AlgebricksException;
- public void step(IFrameTupleReference tuple) throws AlgebricksException;
+ public void step(IFrameTupleReference tuple, IPointable result) throws AlgebricksException;
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IRunningAggregateEvaluatorFactory.java
similarity index 68%
copy from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
copy to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IRunningAggregateEvaluatorFactory.java
index 871fe43..80f6511 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IRunningAggregateEvaluatorFactory.java
@@ -12,10 +12,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
import java.io.Serializable;
-public interface IPrinterFactory extends Serializable {
- public IPrinter createPrinter();
-}
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+
+public interface IRunningAggregateEvaluatorFactory extends Serializable {
+ public IRunningAggregateEvaluator createRunningAggregateEvaluator() throws AlgebricksException;
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunction.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IScalarEvaluator.java
similarity index 70%
copy from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunction.java
copy to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IScalarEvaluator.java
index 5b891f2..e22097a 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunction.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IScalarEvaluator.java
@@ -12,14 +12,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-public interface IUnnestingFunction {
- public void init(IFrameTupleReference tuple) throws AlgebricksException;
-
- public boolean step() throws AlgebricksException;
-
-}
+public interface IScalarEvaluator {
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws AlgebricksException;
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IScalarEvaluatorFactory.java
similarity index 64%
copy from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
copy to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IScalarEvaluatorFactory.java
index 871fe43..e40b96c 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IScalarEvaluatorFactory.java
@@ -12,10 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
import java.io.Serializable;
-public interface IPrinterFactory extends Serializable {
- public IPrinter createPrinter();
-}
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+
+public interface IScalarEvaluatorFactory extends Serializable {
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws AlgebricksException;
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunction.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingEvaluator.java
similarity index 72%
copy from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunction.java
copy to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingEvaluator.java
index 5b891f2..f65192a 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunction.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingEvaluator.java
@@ -12,14 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-public interface IUnnestingFunction {
+public interface IUnnestingEvaluator {
public void init(IFrameTupleReference tuple) throws AlgebricksException;
- public boolean step() throws AlgebricksException;
-
-}
+ public boolean step(IPointable result) throws AlgebricksException;
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingEvaluatorFactory.java
similarity index 64%
copy from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
copy to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingEvaluatorFactory.java
index 871fe43..8bca2b9 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingEvaluatorFactory.java
@@ -12,10 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.algebricks.runtime.base;
import java.io.Serializable;
-public interface IPrinterFactory extends Serializable {
- public IPrinter createPrinter();
-}
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+
+public interface IUnnestingEvaluatorFactory extends Serializable {
+ public IUnnestingEvaluator createUnnestingEvaluator(IHyracksTaskContext ctx) throws AlgebricksException;
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/context/AsterixBTreeRegistry.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/context/AsterixBTreeRegistry.java
similarity index 94%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/context/AsterixBTreeRegistry.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/context/AsterixBTreeRegistry.java
index 4ea64d1..8ab92ee 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/context/AsterixBTreeRegistry.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/context/AsterixBTreeRegistry.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context;
+package edu.uci.ics.hyracks.algebricks.runtime.context;
import java.util.HashMap;
import java.util.concurrent.locks.Lock;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/evaluators/ColumnAccessEvalFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ColumnAccessEvalFactory.java
similarity index 73%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/evaluators/ColumnAccessEvalFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ColumnAccessEvalFactory.java
index 4d8bf96..5f5d19c 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/evaluators/ColumnAccessEvalFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ColumnAccessEvalFactory.java
@@ -12,18 +12,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.evaluators;
+package edu.uci.ics.hyracks.algebricks.runtime.evaluators;
import java.io.DataOutput;
import java.io.IOException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-public class ColumnAccessEvalFactory implements IEvaluatorFactory {
+public class ColumnAccessEvalFactory implements ICopyEvaluatorFactory {
private static final long serialVersionUID = 1L;
@@ -39,8 +39,8 @@
}
@Override
- public IEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
- return new IEvaluator() {
+ public ICopyEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
+ return new ICopyEvaluator() {
private DataOutput out = output.getDataOutput();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/evaluators/ConstantEvalFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ConstantEvalFactory.java
similarity index 70%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/evaluators/ConstantEvalFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ConstantEvalFactory.java
index f49f2ee..c537c93 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/evaluators/ConstantEvalFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ConstantEvalFactory.java
@@ -12,18 +12,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.evaluators;
+package edu.uci.ics.hyracks.algebricks.runtime.evaluators;
import java.io.DataOutput;
import java.io.IOException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-public class ConstantEvalFactory implements IEvaluatorFactory {
+public class ConstantEvalFactory implements ICopyEvaluatorFactory {
private static final long serialVersionUID = 1L;
private byte[] value;
@@ -38,8 +38,8 @@
}
@Override
- public IEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
- return new IEvaluator() {
+ public ICopyEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
+ return new ICopyEvaluator() {
private DataOutput out = output.getDataOutput();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ConstantEvaluatorFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ConstantEvaluatorFactory.java
new file mode 100644
index 0000000..4f7c35a
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ConstantEvaluatorFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.runtime.evaluators;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class ConstantEvaluatorFactory implements IScalarEvaluatorFactory {
+ private static final long serialVersionUID = 1L;
+
+ private byte[] value;
+
+ public ConstantEvaluatorFactory(byte[] value) {
+ this.value = value;
+ }
+
+ @Override
+ public String toString() {
+ return "Constant";
+ }
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
+ return new IScalarEvaluator() {
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws AlgebricksException {
+ result.set(value, 0, value.length);
+ }
+ };
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/TupleFieldEvaluatorFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/TupleFieldEvaluatorFactory.java
new file mode 100644
index 0000000..29328ff
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/TupleFieldEvaluatorFactory.java
@@ -0,0 +1,29 @@
+package edu.uci.ics.hyracks.algebricks.runtime.evaluators;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class TupleFieldEvaluatorFactory implements IScalarEvaluatorFactory {
+ private static final long serialVersionUID = 1L;
+
+ private final int fieldIndex;
+
+ public TupleFieldEvaluatorFactory(int fieldIndex) {
+ this.fieldIndex = fieldIndex;
+ }
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
+ return new IScalarEvaluator() {
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws AlgebricksException {
+ result.set(tuple.getFieldData(fieldIndex), tuple.getFieldStart(fieldIndex),
+ tuple.getFieldLength(fieldIndex));
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/AggregateRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/AggregateRuntimeFactory.java
similarity index 74%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/AggregateRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/AggregateRuntimeFactory.java
index 5184e12..a975195 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/AggregateRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/AggregateRuntimeFactory.java
@@ -12,19 +12,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg;
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunction;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
public class AggregateRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
@@ -32,9 +33,9 @@
private static final long serialVersionUID = 1L;
// private int[] outColumns;
- private IAggregateFunctionFactory[] aggregFactories;
+ private IAggregateEvaluatorFactory[] aggregFactories;
- public AggregateRuntimeFactory(IAggregateFunctionFactory[] aggregFactories) {
+ public AggregateRuntimeFactory(IAggregateEvaluatorFactory[] aggregFactories) {
super(null);
// this.outColumns = outColumns;
this.aggregFactories = aggregFactories;
@@ -55,12 +56,12 @@
}
@Override
- public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context)
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx)
throws AlgebricksException {
return new AbstractOneInputOneOutputOneFramePushRuntime() {
- private IAggregateFunction[] aggregs = new IAggregateFunction[aggregFactories.length];
- private ArrayBackedValueStorage evalOutput = new ArrayBackedValueStorage();
+ private IAggregateEvaluator[] aggregs = new IAggregateEvaluator[aggregFactories.length];
+ private IPointable result = VoidPointable.FACTORY.createPointable();
private ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(aggregs.length);
private boolean first = true;
@@ -70,9 +71,9 @@
try {
if (first) {
first = false;
- initAccessAppendRef(context);
+ initAccessAppendRef(ctx);
for (int i = 0; i < aggregFactories.length; i++) {
- aggregs[i] = aggregFactories[i].createAggregateFunction(evalOutput);
+ aggregs[i] = aggregFactories[i].createAggregateEvaluator(ctx);
}
}
for (int i = 0; i < aggregFactories.length; i++) {
@@ -106,13 +107,12 @@
private void computeAggregate() throws HyracksDataException {
tupleBuilder.reset();
for (int f = 0; f < aggregs.length; f++) {
- evalOutput.reset();
try {
- aggregs[f].finish();
+ aggregs[f].finish(result);
} catch (AlgebricksException e) {
throw new HyracksDataException(e);
}
- tupleBuilder.addField(evalOutput.getBytes(), evalOutput.getStartIndex(), evalOutput.getLength());
+ tupleBuilder.addField(result.getByteArray(), result.getStartOffset(), result.getLength());
}
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java
similarity index 90%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java
index b7b64ab..2ae6f72 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java
@@ -12,16 +12,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg;
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.NestedTupleSourceRuntimeFactory.NestedTupleSourceRuntime;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.NestedTupleSourceRuntimeFactory.NestedTupleSourceRuntime;
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
@@ -51,14 +50,12 @@
public IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDesc,
RecordDescriptor outRecordDescriptor, int[] keys, int[] partialKeys) throws HyracksDataException {
- final RuntimeContext rc = new RuntimeContext();
- rc.setHyracksContext(ctx);
final AggregatorOutput outputWriter = new AggregatorOutput(ctx.getFrameSize(), subplans, keyFieldIdx.length,
decorFieldIdx.length);
final NestedTupleSourceRuntime[] pipelines = new NestedTupleSourceRuntime[subplans.length];
for (int i = 0; i < subplans.length; i++) {
try {
- pipelines[i] = (NestedTupleSourceRuntime) assemblePipeline(subplans[i], outputWriter, rc);
+ pipelines[i] = (NestedTupleSourceRuntime) assemblePipeline(subplans[i], outputWriter, ctx);
} catch (AlgebricksException e) {
throw new HyracksDataException(e);
}
@@ -143,14 +140,14 @@
};
}
- private IFrameWriter assemblePipeline(AlgebricksPipeline subplan, IFrameWriter writer, RuntimeContext rc)
+ private IFrameWriter assemblePipeline(AlgebricksPipeline subplan, IFrameWriter writer, IHyracksTaskContext ctx)
throws AlgebricksException {
// plug the operators
IFrameWriter start = writer;
IPushRuntimeFactory[] runtimeFactories = subplan.getRuntimeFactories();
RecordDescriptor[] recordDescriptors = subplan.getRecordDescriptors();
for (int i = runtimeFactories.length - 1; i >= 0; i--) {
- IPushRuntime newRuntime = runtimeFactories[i].createPushRuntime(rc);
+ IPushRuntime newRuntime = runtimeFactories[i].createPushRuntime(ctx);
newRuntime.setFrameWriter(0, start, recordDescriptors[i]);
if (i > 0) {
newRuntime.setInputRecordDescriptor(0, recordDescriptors[i - 1]);
@@ -164,10 +161,7 @@
}
/**
- *
- *
* We suppose for now, that each subplan only produces one tuple.
- *
*/
private static class AggregatorOutput implements IFrameWriter {
@@ -202,10 +196,8 @@
}
/**
- *
* Since each pipeline only produces one tuple, this method is only
* called by the close method of the pipelines.
- *
*/
@Override
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/SerializableAggregatorDescriptorFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/SerializableAggregatorDescriptorFactory.java
similarity index 88%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/SerializableAggregatorDescriptorFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/SerializableAggregatorDescriptorFactory.java
index ffc8c54..4d5d3d6 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/SerializableAggregatorDescriptorFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/SerializableAggregatorDescriptorFactory.java
@@ -1,10 +1,10 @@
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg;
import java.io.DataOutput;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.ISerializableAggregateFunction;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.ISerializableAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunction;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
@@ -17,9 +17,9 @@
public class SerializableAggregatorDescriptorFactory implements IAggregatorDescriptorFactory {
private static final long serialVersionUID = 1L;
- private ISerializableAggregateFunctionFactory[] aggFactories;
+ private ICopySerializableAggregateFunctionFactory[] aggFactories;
- public SerializableAggregatorDescriptorFactory(ISerializableAggregateFunctionFactory[] aggFactories) {
+ public SerializableAggregatorDescriptorFactory(ICopySerializableAggregateFunctionFactory[] aggFactories) {
this.aggFactories = aggFactories;
}
@@ -34,7 +34,7 @@
*/
return new IAggregatorDescriptor() {
private FrameTupleReference ftr = new FrameTupleReference();
- private ISerializableAggregateFunction[] aggs = new ISerializableAggregateFunction[aggFactories.length];
+ private ICopySerializableAggregateFunction[] aggs = new ICopySerializableAggregateFunction[aggFactories.length];
private int offsetFieldIndex = keys.length;
private int stateFieldLength[] = new int[aggFactories.length];
@@ -81,11 +81,12 @@
int stateTupleIndex, AggregateState state) throws HyracksDataException {
ftr.reset(accessor, tIndex);
int stateTupleStart = stateAccessor.getTupleStartOffset(stateTupleIndex);
+ int fieldSlotLength = stateAccessor.getFieldSlotsLength();
for (int i = 0; i < aggs.length; i++) {
try {
byte[] data = stateAccessor.getBuffer().array();
int start = stateAccessor.getFieldStartOffset(stateTupleIndex, i + keys.length)
- + stateTupleStart;
+ + stateTupleStart + fieldSlotLength;
aggs[i].step(ftr, data, start, stateFieldLength[i]);
} catch (AlgebricksException e) {
throw new HyracksDataException(e);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/SimpleAlgebricksAccumulatingAggregatorFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/SimpleAlgebricksAccumulatingAggregatorFactory.java
similarity index 68%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/SimpleAlgebricksAccumulatingAggregatorFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/SimpleAlgebricksAccumulatingAggregatorFactory.java
index 1743dfc..83925cc 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/SimpleAlgebricksAccumulatingAggregatorFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/SimpleAlgebricksAccumulatingAggregatorFactory.java
@@ -12,18 +12,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunction;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptor;
@@ -32,32 +32,29 @@
public class SimpleAlgebricksAccumulatingAggregatorFactory implements IAggregatorDescriptorFactory {
private static final long serialVersionUID = 1L;
- private IAggregateFunctionFactory[] aggFactories;
+ private IAggregateEvaluatorFactory[] aggFactories;
- public SimpleAlgebricksAccumulatingAggregatorFactory(IAggregateFunctionFactory[] aggFactories, int[] keys,
+ public SimpleAlgebricksAccumulatingAggregatorFactory(IAggregateEvaluatorFactory[] aggFactories, int[] keys,
int[] fdColumns) {
this.aggFactories = aggFactories;
}
- @SuppressWarnings("unchecked")
@Override
- public IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDesc,
+ public IAggregatorDescriptor createAggregator(final IHyracksTaskContext ctx, RecordDescriptor inRecordDesc,
RecordDescriptor outRecordDescriptor, int[] aggKeys, int[] partialKeys) throws HyracksDataException {
return new IAggregatorDescriptor() {
private FrameTupleReference ftr = new FrameTupleReference();
+ private IPointable p = VoidPointable.FACTORY.createPointable();
@Override
public void init(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
AggregateState state) throws HyracksDataException {
- Pair<ArrayBackedValueStorage[], IAggregateFunction[]> aggState = (Pair<ArrayBackedValueStorage[], IAggregateFunction[]>) state.state;
- ArrayBackedValueStorage[] aggOutput = aggState.first;
- IAggregateFunction[] agg = aggState.second;
+ IAggregateEvaluator[] agg = (IAggregateEvaluator[]) state.state;
// initialize aggregate functions
for (int i = 0; i < agg.length; i++) {
- aggOutput[i].reset();
try {
agg[i].init();
} catch (AlgebricksException e) {
@@ -78,8 +75,7 @@
@Override
public void aggregate(IFrameTupleAccessor accessor, int tIndex, IFrameTupleAccessor stateAccessor,
int stateTupleIndex, AggregateState state) throws HyracksDataException {
- Pair<ArrayBackedValueStorage[], IAggregateFunction[]> aggState = (Pair<ArrayBackedValueStorage[], IAggregateFunction[]>) state.state;
- IAggregateFunction[] agg = aggState.second;
+ IAggregateEvaluator[] agg = (IAggregateEvaluator[]) state.state;
ftr.reset(accessor, tIndex);
for (int i = 0; i < agg.length; i++) {
try {
@@ -93,14 +89,11 @@
@Override
public void outputFinalResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
AggregateState state) throws HyracksDataException {
- Pair<ArrayBackedValueStorage[], IAggregateFunction[]> aggState = (Pair<ArrayBackedValueStorage[], IAggregateFunction[]>) state.state;
- ArrayBackedValueStorage[] aggOutput = aggState.first;
- IAggregateFunction[] agg = aggState.second;
+ IAggregateEvaluator[] agg = (IAggregateEvaluator[]) state.state;
for (int i = 0; i < agg.length; i++) {
try {
- agg[i].finish();
- tupleBuilder.addField(aggOutput[i].getBytes(), aggOutput[i].getStartIndex(),
- aggOutput[i].getLength());
+ agg[i].finish(p);
+ tupleBuilder.addField(p.getByteArray(), p.getStartOffset(), p.getLength());
} catch (AlgebricksException e) {
throw new HyracksDataException(e);
}
@@ -109,17 +102,15 @@
@Override
public AggregateState createAggregateStates() {
- IAggregateFunction[] agg = new IAggregateFunction[aggFactories.length];
- ArrayBackedValueStorage[] aggOutput = new ArrayBackedValueStorage[aggFactories.length];
+ IAggregateEvaluator[] agg = new IAggregateEvaluator[aggFactories.length];
for (int i = 0; i < agg.length; i++) {
- aggOutput[i] = new ArrayBackedValueStorage();
try {
- agg[i] = aggFactories[i].createAggregateFunction(aggOutput[i]);
+ agg[i] = aggFactories[i].createAggregateEvaluator(ctx);
} catch (AlgebricksException e) {
throw new IllegalStateException(e);
}
}
- return new AggregateState(new Pair<ArrayBackedValueStorage[], IAggregateFunction[]>(aggOutput, agg));
+ return new AggregateState(agg);
}
@Override
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputOneFramePushRuntime.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputOneFramePushRuntime.java
similarity index 82%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputOneFramePushRuntime.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputOneFramePushRuntime.java
index 5d5e7fb..cd9931b 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputOneFramePushRuntime.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputOneFramePushRuntime.java
@@ -12,11 +12,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.base;
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -34,8 +33,10 @@
@Override
public void close() throws HyracksDataException {
- if (appender.getTupleCount() > 0) {
- FrameUtils.flushFrame(frame, writer);
+ if (!failed) {
+ if (appender.getTupleCount() > 0) {
+ FrameUtils.flushFrame(frame, writer);
+ }
}
writer.close();
appender.reset(frame, true);
@@ -74,18 +75,17 @@
}
}
- protected final void initAccessAppend(RuntimeContext context) {
- IHyracksTaskContext hCtx = context.getHyracksContext();
+ protected final void initAccessAppend(IHyracksTaskContext ctx) {
// if (allocFrame) {
- frame = hCtx.allocateFrame();
- appender = new FrameTupleAppender(hCtx.getFrameSize());
+ frame = ctx.allocateFrame();
+ appender = new FrameTupleAppender(ctx.getFrameSize());
appender.reset(frame, true);
// }
- tAccess = new FrameTupleAccessor(hCtx.getFrameSize(), inputRecordDesc);
+ tAccess = new FrameTupleAccessor(ctx.getFrameSize(), inputRecordDesc);
}
- protected final void initAccessAppendRef(RuntimeContext context) {
- initAccessAppend(context);
+ protected final void initAccessAppendRef(IHyracksTaskContext ctx) {
+ initAccessAppend(ctx);
tRef = new FrameTupleReference();
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java
similarity index 92%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java
index 1034232..22f7df6 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.base;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputRuntimeFactory.java
similarity index 64%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputRuntimeFactory.java
index 23f4f6f..f2c1008 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputRuntimeFactory.java
@@ -12,12 +12,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.base;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
public abstract class AbstractOneInputOneOutputRuntimeFactory implements IPushRuntimeFactory {
@@ -30,11 +30,11 @@
}
@Override
- public IPushRuntime createPushRuntime(RuntimeContext context) throws AlgebricksException {
- return createOneOutputPushRuntime(context);
+ public IPushRuntime createPushRuntime(IHyracksTaskContext ctx) throws AlgebricksException {
+ return createOneOutputPushRuntime(ctx);
}
- public abstract AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(RuntimeContext context)
+ public abstract AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(IHyracksTaskContext ctx)
throws AlgebricksException;
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputPushRuntime.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputPushRuntime.java
similarity index 87%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputPushRuntime.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputPushRuntime.java
index 5660b2c..e600264 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputPushRuntime.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputPushRuntime.java
@@ -12,9 +12,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.base;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
@@ -22,6 +22,7 @@
public abstract class AbstractOneInputPushRuntime implements IPushRuntime {
protected IFrameWriter writer;
protected RecordDescriptor outputRecordDesc;
+ protected boolean failed;
@Override
public void setFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) {
@@ -31,6 +32,7 @@
@Override
public void fail() throws HyracksDataException {
+ failed = true;
writer.fail();
}
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputSinkPushRuntime.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputSinkPushRuntime.java
similarity index 88%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputSinkPushRuntime.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputSinkPushRuntime.java
index 82cf050..0fbe51f 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputSinkPushRuntime.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputSinkPushRuntime.java
@@ -12,9 +12,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.base;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputSourcePushRuntime.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputSourcePushRuntime.java
similarity index 94%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputSourcePushRuntime.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputSourcePushRuntime.java
index 3f1c0f3..1d23d14 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputSourcePushRuntime.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputSourcePushRuntime.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.base;
import java.nio.ByteBuffer;
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/SinkRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/SinkRuntimeFactory.java
similarity index 74%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/SinkRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/SinkRuntimeFactory.java
index 791d9a1..f175367 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/SinkRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/SinkRuntimeFactory.java
@@ -12,14 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.base;
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
public class SinkRuntimeFactory implements IPushRuntimeFactory {
@@ -35,7 +35,7 @@
}
@Override
- public IPushRuntime createPushRuntime(RuntimeContext context) throws AlgebricksException {
+ public IPushRuntime createPushRuntime(IHyracksTaskContext ctx) throws AlgebricksException {
return new AbstractOneInputSinkPushRuntime() {
@Override
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/group/MicroPreClusteredGroupRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/group/MicroPreClusteredGroupRuntimeFactory.java
similarity index 83%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/group/MicroPreClusteredGroupRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/group/MicroPreClusteredGroupRuntimeFactory.java
index b15d63d..388e6c9 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/group/MicroPreClusteredGroupRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/group/MicroPreClusteredGroupRuntimeFactory.java
@@ -1,12 +1,11 @@
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.group;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.group;
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputPushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputPushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -16,7 +15,7 @@
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupWriter;
+import edu.uci.ics.hyracks.dataflow.std.group.preclustered.PreclusteredGroupWriter;
public class MicroPreClusteredGroupRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
@@ -43,14 +42,13 @@
}
@Override
- public AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(RuntimeContext context)
+ public AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx)
throws AlgebricksException {
try {
final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
for (int i = 0; i < comparatorFactories.length; ++i) {
comparators[i] = comparatorFactories[i].createBinaryComparator();
}
- final IHyracksTaskContext ctx = context.getHyracksContext();
final IAggregatorDescriptor aggregator = aggregatorFactory.createAggregator(ctx, inRecordDesc,
outRecordDesc, groupFields, groupFields);
final ByteBuffer copyFrame = ctx.allocateFrame();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java
similarity index 87%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java
index 2fdeb23..f2659f5 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java
@@ -12,24 +12,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.meta;
import java.nio.ByteBuffer;
import org.json.JSONException;
import org.json.JSONObject;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
@@ -41,7 +40,7 @@
// array of factories for building the local runtime pipeline
private final AlgebricksPipeline pipeline;
- public AlgebricksMetaOperatorDescriptor(JobSpecification spec, int inputArity, int outputArity,
+ public AlgebricksMetaOperatorDescriptor(IOperatorDescriptorRegistry spec, int inputArity, int outputArity,
IPushRuntimeFactory[] runtimeFactories, RecordDescriptor[] internalRecordDescriptors) {
super(spec, inputArity, outputArity);
if (outputArity == 1) {
@@ -98,9 +97,7 @@
PipelineAssembler pa = new PipelineAssembler(pipeline, inputArity, outputArity, null,
pipelineOutputRecordDescriptor);
try {
- RuntimeContext rc = new RuntimeContext();
- rc.setHyracksContext(ctx);
- startOfPipeline = pa.assemblePipeline(writer, rc);
+ startOfPipeline = pa.assemblePipeline(writer, ctx);
} catch (AlgebricksException e) {
throw new HyracksDataException(e);
}
@@ -122,13 +119,11 @@
RecordDescriptor pipelineOutputRecordDescriptor = outputArity > 0 ? AlgebricksMetaOperatorDescriptor.this.recordDescriptors[0]
: null;
RecordDescriptor pipelineInputRecordDescriptor = recordDescProvider.getInputRecordDescriptor(
- AlgebricksMetaOperatorDescriptor.this.getOperatorId(), 0);
+ AlgebricksMetaOperatorDescriptor.this.getActivityId(), 0);
PipelineAssembler pa = new PipelineAssembler(pipeline, inputArity, outputArity,
pipelineInputRecordDescriptor, pipelineOutputRecordDescriptor);
try {
- RuntimeContext rc = new RuntimeContext();
- rc.setHyracksContext(ctx);
- startOfPipeline = pa.assemblePipeline(writer, rc);
+ startOfPipeline = pa.assemblePipeline(writer, ctx);
} catch (AlgebricksException ae) {
throw new HyracksDataException(ae);
}
@@ -148,7 +143,8 @@
@Override
public void fail() throws HyracksDataException {
+ startOfPipeline.fail();
}
};
}
-}
\ No newline at end of file
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/PipelineAssembler.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/PipelineAssembler.java
similarity index 81%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/PipelineAssembler.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/PipelineAssembler.java
index f0f5029..92410c2 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/PipelineAssembler.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/PipelineAssembler.java
@@ -12,13 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.meta;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
public class PipelineAssembler {
@@ -40,11 +40,11 @@
this.outputArity = outputArity;
}
- public IFrameWriter assemblePipeline(IFrameWriter writer, RuntimeContext rc) throws AlgebricksException {
+ public IFrameWriter assemblePipeline(IFrameWriter writer, IHyracksTaskContext ctx) throws AlgebricksException {
// plug the operators
IFrameWriter start = writer;// this.writer;
for (int i = pipeline.getRuntimeFactories().length - 1; i >= 0; i--) {
- IPushRuntime newRuntime = pipeline.getRuntimeFactories()[i].createPushRuntime(rc);
+ IPushRuntime newRuntime = pipeline.getRuntimeFactories()[i].createPushRuntime(ctx);
if (i == pipeline.getRuntimeFactories().length - 1) {
if (outputArity == 1) {
newRuntime.setFrameWriter(0, start, pipelineOutputRecordDescriptor);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/SubplanRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/SubplanRuntimeFactory.java
similarity index 83%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/SubplanRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/SubplanRuntimeFactory.java
index 53808e1..f1ee570 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/SubplanRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/SubplanRuntimeFactory.java
@@ -12,20 +12,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.meta;
import java.io.DataOutput;
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputPushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.NestedTupleSourceRuntimeFactory.NestedTupleSourceRuntime;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputPushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.NestedTupleSourceRuntimeFactory.NestedTupleSourceRuntime;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.INullWriter;
@@ -67,7 +66,7 @@
}
@Override
- public AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(final RuntimeContext context)
+ public AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx)
throws AlgebricksException {
RecordDescriptor pipelineOutputRecordDescriptor = null;
@@ -82,15 +81,13 @@
return new AbstractOneInputOneOutputOneFramePushRuntime() {
/**
- *
* Computes the outer product between a given tuple and the frames
* passed.
- *
*/
class TupleOuterProduct implements IFrameWriter {
private boolean smthWasWritten = false;
- private IHyracksTaskContext hCtx = context.getHyracksContext();
+ private IHyracksTaskContext hCtx = ctx;
private int frameSize = hCtx.getFrameSize();
private FrameTupleAccessor ta = new FrameTupleAccessor(frameSize,
pipeline.getRecordDescriptors()[pipeline.getRecordDescriptors().length - 1]);
@@ -148,7 +145,7 @@
IFrameWriter endPipe = new TupleOuterProduct();
- NestedTupleSourceRuntime startOfPipeline = (NestedTupleSourceRuntime) pa.assemblePipeline(endPipe, context);
+ NestedTupleSourceRuntime startOfPipeline = (NestedTupleSourceRuntime) pa.assemblePipeline(endPipe, ctx);
boolean first = true;
@@ -156,7 +153,7 @@
public void open() throws HyracksDataException {
if (first) {
first = false;
- initAccessAppendRef(context);
+ initAccessAppendRef(ctx);
}
writer.open();
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/sort/InMemorySortRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/sort/InMemorySortRuntimeFactory.java
similarity index 78%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/sort/InMemorySortRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/sort/InMemorySortRuntimeFactory.java
index 2a6c196..fd2710d 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/sort/InMemorySortRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/sort/InMemorySortRuntimeFactory.java
@@ -12,15 +12,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.sort;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.sort;
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputPushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputPushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
@@ -47,7 +47,7 @@
}
@Override
- public AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(final RuntimeContext context)
+ public AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx)
throws AlgebricksException {
return new AbstractOneInputOneOutputPushRuntime() {
@@ -57,8 +57,8 @@
@Override
public void open() throws HyracksDataException {
if (frameSorter == null) {
- frameSorter = new FrameSorter(context.getHyracksContext(), sortFields, firstKeyNormalizerFactory,
- comparatorFactories, outputRecordDesc);
+ frameSorter = new FrameSorter(ctx, sortFields, firstKeyNormalizerFactory, comparatorFactories,
+ outputRecordDesc);
}
frameSorter.reset();
writer.open();
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/AssignRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/AssignRuntimeFactory.java
similarity index 75%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/AssignRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/AssignRuntimeFactory.java
index 04dc7d6..e47384d 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/AssignRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/AssignRuntimeFactory.java
@@ -12,21 +12,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
import java.nio.ByteBuffer;
import java.util.Arrays;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
public class AssignRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
@@ -34,10 +35,9 @@
private static final long serialVersionUID = 1L;
private int[] outColumns;
- private IEvaluatorFactory[] evalFactories;
+ private IScalarEvaluatorFactory[] evalFactories;
/**
- *
* @param outColumns
* a sorted array of columns into which the result is written to
* @param evalFactories
@@ -45,7 +45,7 @@
* an array of columns to be projected
*/
- public AssignRuntimeFactory(int[] outColumns, IEvaluatorFactory[] evalFactories, int[] projectionList) {
+ public AssignRuntimeFactory(int[] outColumns, IScalarEvaluatorFactory[] evalFactories, int[] projectionList) {
super(projectionList);
this.outColumns = outColumns;
this.evalFactories = evalFactories;
@@ -73,7 +73,7 @@
}
@Override
- public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context)
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx)
throws AlgebricksException {
final int[] projectionToOutColumns = new int[projectionList.length];
for (int j = 0; j < projectionList.length; j++) {
@@ -81,21 +81,20 @@
}
return new AbstractOneInputOneOutputOneFramePushRuntime() {
-
- private ArrayBackedValueStorage evalOutput = new ArrayBackedValueStorage();
- private IEvaluator[] eval = new IEvaluator[evalFactories.length];
+ private IPointable result = VoidPointable.FACTORY.createPointable();
+ private IScalarEvaluator[] eval = new IScalarEvaluator[evalFactories.length];
private ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(projectionList.length);
private boolean first = true;
@Override
public void open() throws HyracksDataException {
if (first) {
- initAccessAppendRef(context);
+ initAccessAppendRef(ctx);
first = false;
int n = evalFactories.length;
for (int i = 0; i < n; i++) {
try {
- eval[i] = evalFactories[i].createEvaluator(evalOutput);
+ eval[i] = evalFactories[i].createScalarEvaluator(ctx);
} catch (AlgebricksException ae) {
throw new HyracksDataException(ae);
}
@@ -121,13 +120,12 @@
for (int f = 0; f < projectionList.length; f++) {
int k = projectionToOutColumns[f];
if (k >= 0) {
- evalOutput.reset();
try {
- eval[k].evaluate(tupleRef);
+ eval[k].evaluate(tupleRef, result);
} catch (AlgebricksException e) {
throw new HyracksDataException(e);
}
- tb.addField(evalOutput.getBytes(), evalOutput.getStartIndex(), evalOutput.getLength());
+ tb.addField(result.getByteArray(), result.getStartOffset(), result.getLength());
} else {
tb.addField(accessor, tIndex, projectionList[f]);
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/EmptyTupleSourceRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/EmptyTupleSourceRuntimeFactory.java
similarity index 73%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/EmptyTupleSourceRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/EmptyTupleSourceRuntimeFactory.java
index 2d0a038..dbe1ab1 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/EmptyTupleSourceRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/EmptyTupleSourceRuntimeFactory.java
@@ -12,14 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputSourcePushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputSourcePushRuntime;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -39,13 +38,12 @@
}
@Override
- public IPushRuntime createPushRuntime(final RuntimeContext context) {
+ public IPushRuntime createPushRuntime(final IHyracksTaskContext ctx) {
return new AbstractOneInputSourcePushRuntime() {
- private IHyracksTaskContext hCtx = context.getHyracksContext();
- private ByteBuffer frame = hCtx.allocateFrame();
+ private ByteBuffer frame = ctx.allocateFrame();
private ArrayTupleBuilder tb = new ArrayTupleBuilder(0);
- private FrameTupleAppender appender = new FrameTupleAppender(hCtx.getFrameSize());
+ private FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
@Override
public void open() throws HyracksDataException {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/NestedTupleSourceRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/NestedTupleSourceRuntimeFactory.java
similarity index 72%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/NestedTupleSourceRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/NestedTupleSourceRuntimeFactory.java
index 8c650b5..42724f7 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/NestedTupleSourceRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/NestedTupleSourceRuntimeFactory.java
@@ -12,14 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
public class NestedTupleSourceRuntimeFactory implements IPushRuntimeFactory {
@@ -35,14 +35,14 @@
}
@Override
- public IPushRuntime createPushRuntime(RuntimeContext context) {
- return new NestedTupleSourceRuntime(context);
+ public IPushRuntime createPushRuntime(IHyracksTaskContext ctx) {
+ return new NestedTupleSourceRuntime(ctx);
}
public static class NestedTupleSourceRuntime extends AbstractOneInputOneOutputOneFramePushRuntime {
- public NestedTupleSourceRuntime(RuntimeContext rc) {
- initAccessAppend(rc);
+ public NestedTupleSourceRuntime(IHyracksTaskContext ctx) {
+ initAccessAppend(ctx);
}
@Override
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/PartitioningSplitOperatorDescriptor.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/PartitioningSplitOperatorDescriptor.java
new file mode 100644
index 0000000..a54c96d
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/PartitioningSplitOperatorDescriptor.java
@@ -0,0 +1,160 @@
+package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspector;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputOperatorNodePushable;
+
+public class PartitioningSplitOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+ private static final long serialVersionUID = 1L;
+ public static int NO_DEFAULT_BRANCH = -1;
+
+ private final ICopyEvaluatorFactory[] evalFactories;
+ private final IBinaryBooleanInspector boolInspector;
+ private final int defaultBranchIndex;
+
+ public PartitioningSplitOperatorDescriptor(IOperatorDescriptorRegistry spec, ICopyEvaluatorFactory[] evalFactories,
+ IBinaryBooleanInspector boolInspector, int defaultBranchIndex, RecordDescriptor rDesc) {
+ super(spec, 1, (defaultBranchIndex == evalFactories.length) ? evalFactories.length + 1 : evalFactories.length);
+ for (int i = 0; i < evalFactories.length; i++) {
+ recordDescriptors[i] = rDesc;
+ }
+ this.evalFactories = evalFactories;
+ this.boolInspector = boolInspector;
+ this.defaultBranchIndex = defaultBranchIndex;
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ final IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions)
+ throws HyracksDataException {
+ return new AbstractUnaryInputOperatorNodePushable() {
+ private final IFrameWriter[] writers = new IFrameWriter[outputArity];
+ private final ByteBuffer[] writeBuffers = new ByteBuffer[outputArity];
+ private final ICopyEvaluator[] evals = new ICopyEvaluator[outputArity];
+ private final ArrayBackedValueStorage evalBuf = new ArrayBackedValueStorage();
+ private final RecordDescriptor inOutRecDesc = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);
+ private final FrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), inOutRecDesc);
+ private final FrameTupleReference frameTuple = new FrameTupleReference();
+
+ private final FrameTupleAppender tupleAppender = new FrameTupleAppender(ctx.getFrameSize());
+ private final ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(inOutRecDesc.getFieldCount());
+ private final DataOutput tupleDos = tupleBuilder.getDataOutput();
+
+ @Override
+ public void close() throws HyracksDataException {
+ // Flush (possibly not full) buffers that have data, and close writers.
+ for (int i = 0; i < outputArity; i++) {
+ tupleAppender.reset(writeBuffers[i], false);
+ if (tupleAppender.getTupleCount() > 0) {
+ FrameUtils.flushFrame(writeBuffers[i], writers[i]);
+ }
+ writers[i].close();
+ }
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ for (IFrameWriter writer : writers) {
+ writer.fail();
+ }
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ accessor.reset(buffer);
+ int tupleCount = accessor.getTupleCount();
+ for (int i = 0; i < tupleCount; i++) {
+ frameTuple.reset(accessor, i);
+ boolean found = false;
+ for (int j = 0; j < evals.length; j++) {
+ try {
+ evalBuf.reset();
+ evals[j].evaluate(frameTuple);
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ found = boolInspector.getBooleanValue(evalBuf.getByteArray(), 0, 1);
+ if (found) {
+ copyAndAppendTuple(j);
+ break;
+ }
+ }
+ // Optionally write to default output branch.
+ if (!found && defaultBranchIndex != NO_DEFAULT_BRANCH) {
+ copyAndAppendTuple(defaultBranchIndex);
+ }
+ }
+ }
+
+ private void copyAndAppendTuple(int outputIndex) throws HyracksDataException {
+ // Copy tuple into tuple builder.
+ try {
+ tupleBuilder.reset();
+ for (int i = 0; i < frameTuple.getFieldCount(); i++) {
+ tupleDos.write(frameTuple.getFieldData(i), frameTuple.getFieldStart(i),
+ frameTuple.getFieldLength(i));
+ tupleBuilder.addFieldEndOffset();
+ }
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ // Append to frame.
+ tupleAppender.reset(writeBuffers[outputIndex], false);
+ if (!tupleAppender.append(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray(), 0, tupleBuilder.getSize())) {
+ FrameUtils.flushFrame(writeBuffers[outputIndex], writers[outputIndex]);
+ tupleAppender.reset(writeBuffers[outputIndex], true);
+ if (!tupleAppender.append(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray(), 0, tupleBuilder.getSize())) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ for (IFrameWriter writer : writers) {
+ writer.open();
+ }
+ // Create write buffers.
+ for (int i = 0; i < outputArity; i++) {
+ writeBuffers[i] = ctx.allocateFrame();
+ // Make sure to clear all buffers, since we are reusing the tupleAppender.
+ tupleAppender.reset(writeBuffers[i], true);
+ }
+ // Create evaluators for partitioning.
+ try {
+ for (int i = 0; i < evalFactories.length; i++) {
+ evals[i] = evalFactories[i].createEvaluator(evalBuf);
+ }
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) {
+ writers[index] = writer;
+ }
+ };
+ }
+}
+
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/PrinterRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/PrinterRuntimeFactory.java
similarity index 71%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/PrinterRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/PrinterRuntimeFactory.java
index 787e001..5b53fc2 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/PrinterRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/PrinterRuntimeFactory.java
@@ -12,14 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IAWriter;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.writers.PrinterBasedWriterFactory;
+import edu.uci.ics.hyracks.algebricks.data.IAWriter;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.writers.PrinterBasedWriterFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
public class PrinterRuntimeFactory implements IPushRuntimeFactory {
@@ -52,9 +52,9 @@
}
@Override
- public IPushRuntime createPushRuntime(final RuntimeContext context) {
+ public IPushRuntime createPushRuntime(IHyracksTaskContext ctx) {
IAWriter w = PrinterBasedWriterFactory.INSTANCE.createWriter(printColumns, System.out, printerFactories,
inputRecordDesc);
- return new SinkWriterRuntime(w, context, System.out, inputRecordDesc);
+ return new SinkWriterRuntime(w, ctx, System.out, inputRecordDesc);
}
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/RunningAggregateRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/RunningAggregateRuntimeFactory.java
similarity index 76%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/RunningAggregateRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/RunningAggregateRuntimeFactory.java
index bca8f2f..c15e050 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/RunningAggregateRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/RunningAggregateRuntimeFactory.java
@@ -12,21 +12,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
import java.nio.ByteBuffer;
import java.util.Arrays;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IRunningAggregateFunction;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IRunningAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
public class RunningAggregateRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
@@ -34,10 +35,9 @@
private static final long serialVersionUID = 1L;
private int[] outColumns;
- private IRunningAggregateFunctionFactory[] runningAggregates;
+ private IRunningAggregateEvaluatorFactory[] runningAggregates;
/**
- *
* @param outColumns
* a sorted array of columns into which the result is written to
* @param runningAggregates
@@ -45,7 +45,7 @@
* an array of columns to be projected
*/
- public RunningAggregateRuntimeFactory(int[] outColumns, IRunningAggregateFunctionFactory[] runningAggregates,
+ public RunningAggregateRuntimeFactory(int[] outColumns, IRunningAggregateEvaluatorFactory[] runningAggregates,
int[] projectionList) {
super(projectionList);
this.outColumns = outColumns;
@@ -74,7 +74,7 @@
}
@Override
- public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context)
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx)
throws AlgebricksException {
final int[] projectionToOutColumns = new int[projectionList.length];
for (int j = 0; j < projectionList.length; j++) {
@@ -82,21 +82,20 @@
}
return new AbstractOneInputOneOutputOneFramePushRuntime() {
-
- private ArrayBackedValueStorage evalOutput = new ArrayBackedValueStorage();
- private IRunningAggregateFunction[] raggs = new IRunningAggregateFunction[runningAggregates.length];
+ private IPointable p = VoidPointable.FACTORY.createPointable();
+ private IRunningAggregateEvaluator[] raggs = new IRunningAggregateEvaluator[runningAggregates.length];
private ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(projectionList.length);
private boolean first = true;
@Override
public void open() throws HyracksDataException {
- initAccessAppendRef(context);
+ initAccessAppendRef(ctx);
if (first) {
first = false;
int n = runningAggregates.length;
for (int i = 0; i < n; i++) {
try {
- raggs[i] = runningAggregates[i].createRunningAggregateFunction(evalOutput);
+ raggs[i] = runningAggregates[i].createRunningAggregateEvaluator();
raggs[i].init();
} catch (AlgebricksException ae) {
throw new HyracksDataException(ae);
@@ -123,13 +122,12 @@
for (int f = 0; f < projectionList.length; f++) {
int k = projectionToOutColumns[f];
if (k >= 0) {
- evalOutput.reset();
try {
- raggs[k].step(tupleRef);
+ raggs[k].step(tupleRef, p);
} catch (AlgebricksException e) {
throw new HyracksDataException(e);
}
- tb.addField(evalOutput.getBytes(), evalOutput.getStartIndex(), evalOutput.getLength());
+ tb.addField(p.getByteArray(), p.getStartOffset(), p.getLength());
} else {
tb.addField(accessor, tIndex, projectionList[f]);
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/SinkWriterRuntime.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/SinkWriterRuntime.java
similarity index 72%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/SinkWriterRuntime.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/SinkWriterRuntime.java
index 8101a8a..148f087 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/SinkWriterRuntime.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/SinkWriterRuntime.java
@@ -12,22 +12,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
import java.io.PrintStream;
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IAWriter;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputSinkPushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IAWriter;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputSinkPushRuntime;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
public class SinkWriterRuntime extends AbstractOneInputSinkPushRuntime {
- private final RuntimeContext context;
+ private final IHyracksTaskContext ctx;
private final PrintStream printStream;
private final IAWriter writer;
private RecordDescriptor inputRecordDesc;
@@ -35,18 +35,18 @@
private boolean autoClose = false;
private boolean first = true;
- public SinkWriterRuntime(IAWriter writer, RuntimeContext context, PrintStream printStream,
+ public SinkWriterRuntime(IAWriter writer, IHyracksTaskContext ctx, PrintStream printStream,
RecordDescriptor inputRecordDesc) {
this.writer = writer;
- this.context = context;
+ this.ctx = ctx;
this.printStream = printStream;
this.inputRecordDesc = inputRecordDesc;
- this.tAccess = new FrameTupleAccessor(context.getHyracksContext().getFrameSize(), inputRecordDesc);
+ this.tAccess = new FrameTupleAccessor(ctx.getFrameSize(), inputRecordDesc);
}
- public SinkWriterRuntime(IAWriter writer, RuntimeContext context, PrintStream printStream,
+ public SinkWriterRuntime(IAWriter writer, IHyracksTaskContext ctx, PrintStream printStream,
RecordDescriptor inputRecordDesc, boolean autoClose) {
- this(writer, context, printStream, inputRecordDesc);
+ this(writer, ctx, printStream, inputRecordDesc);
this.autoClose = autoClose;
}
@@ -54,7 +54,7 @@
public void open() throws HyracksDataException {
if (first) {
first = false;
- tAccess = new FrameTupleAccessor(context.getHyracksContext().getFrameSize(), inputRecordDesc);
+ tAccess = new FrameTupleAccessor(ctx.getFrameSize(), inputRecordDesc);
try {
writer.init();
} catch (AlgebricksException e) {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/SinkWriterRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/SinkWriterRuntimeFactory.java
similarity index 74%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/SinkWriterRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/SinkWriterRuntimeFactory.java
index 4ac1d16..1aff1aa 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/SinkWriterRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/SinkWriterRuntimeFactory.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
import java.io.BufferedOutputStream;
import java.io.File;
@@ -20,13 +20,13 @@
import java.io.FileOutputStream;
import java.io.PrintStream;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IAWriter;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IAWriterFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IAWriter;
+import edu.uci.ics.hyracks.algebricks.data.IAWriterFactory;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
public class SinkWriterRuntimeFactory implements IPushRuntimeFactory {
@@ -63,7 +63,7 @@
}
@Override
- public IPushRuntime createPushRuntime(RuntimeContext context) throws AlgebricksException {
+ public IPushRuntime createPushRuntime(IHyracksTaskContext ctx) throws AlgebricksException {
PrintStream filePrintStream = null;
try {
filePrintStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outputFile)));
@@ -71,6 +71,6 @@
throw new AlgebricksException(e);
}
IAWriter w = writerFactory.createWriter(fields, filePrintStream, printerFactories, inputRecordDesc);
- return new SinkWriterRuntime(w, context, filePrintStream, inputRecordDesc, true);
+ return new SinkWriterRuntime(w, ctx, filePrintStream, inputRecordDesc, true);
}
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamDieRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamDieRuntimeFactory.java
new file mode 100644
index 0000000..796ef0a
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamDieRuntimeFactory.java
@@ -0,0 +1,98 @@
+package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspector;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspectorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
+
+public class StreamDieRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private IScalarEvaluatorFactory aftterObjectsEvalFactory;
+ private IBinaryIntegerInspectorFactory binaryIntegerInspectorFactory;
+
+ public StreamDieRuntimeFactory(IScalarEvaluatorFactory maxObjectsEvalFactory, int[] projectionList,
+ IBinaryIntegerInspectorFactory binaryIntegerInspectorFactory) {
+ super(projectionList);
+ this.aftterObjectsEvalFactory = maxObjectsEvalFactory;
+ this.binaryIntegerInspectorFactory = binaryIntegerInspectorFactory;
+ }
+
+ @Override
+ public String toString() {
+ String s = "stream-die " + aftterObjectsEvalFactory.toString();
+ return s;
+ }
+
+ @Override
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx) {
+ final IBinaryIntegerInspector bii = binaryIntegerInspectorFactory.createBinaryIntegerInspector(ctx);
+ return new AbstractOneInputOneOutputOneFramePushRuntime() {
+ private IPointable p = VoidPointable.FACTORY.createPointable();
+ private IScalarEvaluator evalAfterObjects;
+ private int toWrite = -1;
+
+ @Override
+ public void open() throws HyracksDataException {
+ if (evalAfterObjects == null) {
+ initAccessAppendRef(ctx);
+ try {
+ evalAfterObjects = aftterObjectsEvalFactory.createScalarEvaluator(ctx);
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ }
+ writer.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ tAccess.reset(buffer);
+ int nTuple = tAccess.getTupleCount();
+ if (toWrite < 0) {
+ toWrite = evaluateInteger(evalAfterObjects, 0);
+ }
+ for (int t = 0; t < nTuple; t++) {
+ if (toWrite > 0) {
+ toWrite--;
+ if (projectionList != null) {
+ appendProjectionToFrame(t, projectionList);
+ } else {
+ appendTupleToFrame(t);
+ }
+ } else {
+ throw new HyracksDataException("injected failure");
+ }
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ super.close();
+ }
+
+ private int evaluateInteger(IScalarEvaluator eval, int tIdx) throws HyracksDataException {
+ tRef.reset(tAccess, tIdx);
+ try {
+ eval.evaluate(tRef, p);
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ int lim = bii.getIntegerValue(p.getByteArray(), p.getStartOffset(), p.getLength());
+ return lim;
+ }
+
+ };
+ }
+
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamLimitRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamLimitRuntimeFactory.java
similarity index 64%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamLimitRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamLimitRuntimeFactory.java
index 5caea6e..4e9d51c 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamLimitRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamLimitRuntimeFactory.java
@@ -1,31 +1,34 @@
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryIntegerInspector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspector;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspectorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
public class StreamLimitRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
private static final long serialVersionUID = 1L;
- private IEvaluatorFactory maxObjectsEvalFactory;
- private IEvaluatorFactory offsetEvalFactory;
- private IBinaryIntegerInspector binaryIntegerInspector;
+ private IScalarEvaluatorFactory maxObjectsEvalFactory;
+ private IScalarEvaluatorFactory offsetEvalFactory;
+ private IBinaryIntegerInspectorFactory binaryIntegerInspectorFactory;
- public StreamLimitRuntimeFactory(IEvaluatorFactory maxObjectsEvalFactory, IEvaluatorFactory offsetEvalFactory,
- int[] projectionList, IBinaryIntegerInspector binaryIntegerInspector) {
+ public StreamLimitRuntimeFactory(IScalarEvaluatorFactory maxObjectsEvalFactory,
+ IScalarEvaluatorFactory offsetEvalFactory, int[] projectionList,
+ IBinaryIntegerInspectorFactory binaryIntegerInspectorFactory) {
super(projectionList);
this.maxObjectsEvalFactory = maxObjectsEvalFactory;
this.offsetEvalFactory = offsetEvalFactory;
- this.binaryIntegerInspector = binaryIntegerInspector;
+ this.binaryIntegerInspectorFactory = binaryIntegerInspectorFactory;
}
@Override
@@ -39,12 +42,12 @@
}
@Override
- public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context) {
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx) {
+ final IBinaryIntegerInspector bii = binaryIntegerInspectorFactory.createBinaryIntegerInspector(ctx);
return new AbstractOneInputOneOutputOneFramePushRuntime() {
-
- private IEvaluator evalMaxObjects;
- private ArrayBackedValueStorage evalOutput;
- private IEvaluator evalOffset = null;
+ private IPointable p = VoidPointable.FACTORY.createPointable();
+ private IScalarEvaluator evalMaxObjects;
+ private IScalarEvaluator evalOffset = null;
private int toWrite = 0; // how many tuples still to write
private int toSkip = 0; // how many tuples still to skip
private boolean firstTuple = true;
@@ -54,12 +57,11 @@
public void open() throws HyracksDataException {
// if (first) {
if (evalMaxObjects == null) {
- initAccessAppendRef(context);
- evalOutput = new ArrayBackedValueStorage();
+ initAccessAppendRef(ctx);
try {
- evalMaxObjects = maxObjectsEvalFactory.createEvaluator(evalOutput);
+ evalMaxObjects = maxObjectsEvalFactory.createScalarEvaluator(ctx);
if (offsetEvalFactory != null) {
- evalOffset = offsetEvalFactory.createEvaluator(evalOutput);
+ evalOffset = offsetEvalFactory.createScalarEvaluator(ctx);
}
} catch (AlgebricksException ae) {
throw new HyracksDataException(ae);
@@ -117,15 +119,14 @@
// }
}
- private int evaluateInteger(IEvaluator eval, int tIdx) throws HyracksDataException {
+ private int evaluateInteger(IScalarEvaluator eval, int tIdx) throws HyracksDataException {
tRef.reset(tAccess, tIdx);
- evalOutput.reset();
try {
- eval.evaluate(tRef);
+ eval.evaluate(tRef, p);
} catch (AlgebricksException ae) {
throw new HyracksDataException(ae);
}
- int lim = binaryIntegerInspector.getIntegerValue(evalOutput.getBytes(), 0, evalOutput.getLength());
+ int lim = bii.getIntegerValue(p.getByteArray(), p.getStartOffset(), p.getLength());
return lim;
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamProjectRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamProjectRuntimeFactory.java
similarity index 77%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamProjectRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamProjectRuntimeFactory.java
index fe30701..455ad8e 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamProjectRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamProjectRuntimeFactory.java
@@ -12,15 +12,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
import java.nio.ByteBuffer;
import java.util.Arrays;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
public class StreamProjectRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
@@ -37,7 +37,7 @@
}
@Override
- public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context)
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx)
throws AlgebricksException {
return new AbstractOneInputOneOutputOneFramePushRuntime() {
@@ -48,7 +48,7 @@
public void open() throws HyracksDataException {
if (first) {
first = false;
- initAccessAppend(context);
+ initAccessAppend(ctx);
}
writer.open();
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamSelectRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamSelectRuntimeFactory.java
new file mode 100644
index 0000000..b0c94b7
--- /dev/null
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamSelectRuntimeFactory.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspector;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
+
+public class StreamSelectRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private IScalarEvaluatorFactory cond;
+
+ private IBinaryBooleanInspectorFactory binaryBooleanInspectorFactory;
+
+ /**
+ * @param cond
+ * @param projectionList
+ * if projectionList is null, then no projection is performed
+ */
+ public StreamSelectRuntimeFactory(IScalarEvaluatorFactory cond, int[] projectionList,
+ IBinaryBooleanInspectorFactory binaryBooleanInspectorFactory) {
+ super(projectionList);
+ this.cond = cond;
+ this.binaryBooleanInspectorFactory = binaryBooleanInspectorFactory;
+ }
+
+ @Override
+ public String toString() {
+ return "stream-select " + cond.toString();
+ }
+
+ @Override
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx) {
+ final IBinaryBooleanInspector bbi = binaryBooleanInspectorFactory.createBinaryBooleanInspector(ctx);
+ return new AbstractOneInputOneOutputOneFramePushRuntime() {
+ private IPointable p = VoidPointable.FACTORY.createPointable();
+ private IScalarEvaluator eval;
+
+ @Override
+ public void open() throws HyracksDataException {
+ if (eval == null) {
+ initAccessAppendRef(ctx);
+ try {
+ eval = cond.createScalarEvaluator(ctx);
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ }
+ writer.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ tAccess.reset(buffer);
+ int nTuple = tAccess.getTupleCount();
+ for (int t = 0; t < nTuple; t++) {
+ tRef.reset(tAccess, t);
+ try {
+ eval.evaluate(tRef, p);
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ if (bbi.getBooleanValue(p.getByteArray(), p.getStartOffset(), p.getLength())) {
+ if (projectionList != null) {
+ appendProjectionToFrame(t, projectionList);
+ } else {
+ appendTupleToFrame(t);
+ }
+ }
+ }
+ }
+
+ };
+ }
+
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StringStreamingRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StringStreamingRuntimeFactory.java
similarity index 89%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StringStreamingRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StringStreamingRuntimeFactory.java
index f3d07b1..3f2765f 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StringStreamingRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StringStreamingRuntimeFactory.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
import java.io.BufferedReader;
import java.io.IOException;
@@ -21,12 +21,12 @@
import java.io.PrintStream;
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinter;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IPrinter;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.std.file.ITupleParser;
import edu.uci.ics.hyracks.dataflow.std.file.ITupleParserFactory;
@@ -50,7 +50,7 @@
}
@Override
- public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context)
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx)
throws AlgebricksException {
final IPrinter[] printers = new IPrinter[printerFactories.length];
for (int i = 0; i < printerFactories.length; i++) {
@@ -125,11 +125,11 @@
public void open() throws HyracksDataException {
if (first) {
first = false;
- initAccessAppendRef(context);
+ initAccessAppendRef(ctx);
}
try {
- ITupleParser parser = parserFactory.createTupleParser(context.getHyracksContext());
+ ITupleParser parser = parserFactory.createTupleParser(ctx);
process = Runtime.getRuntime().exec(command);
ps = new PrintStream(process.getOutputStream());
ForwardScriptOutput fso = new ForwardScriptOutput(parser, process.getInputStream());
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/UnnestRuntimeFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/UnnestRuntimeFactory.java
similarity index 71%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/UnnestRuntimeFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/UnnestRuntimeFactory.java
index bc55b6a..d42a294 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/UnnestRuntimeFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/UnnestRuntimeFactory.java
@@ -12,26 +12,27 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IUnnestingFunction;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IUnnestingFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
public class UnnestRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
private static final long serialVersionUID = 1L;
private final int outCol;
- private final IUnnestingFunctionFactory unnestingFactory;
+ private final IUnnestingEvaluatorFactory unnestingFactory;
private int outColPos;
private final boolean outColIsProjected;
@@ -40,7 +41,7 @@
// By convention, if the aggregate function writes nothing, it means it
// produced the last value.
- public UnnestRuntimeFactory(int outCol, IUnnestingFunctionFactory unnestingFactory, int[] projectionList) {
+ public UnnestRuntimeFactory(int outCol, IUnnestingEvaluatorFactory unnestingFactory, int[] projectionList) {
super(projectionList);
this.outCol = outCol;
this.unnestingFactory = unnestingFactory;
@@ -59,21 +60,19 @@
}
@Override
- public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context)
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx)
throws AlgebricksException {
return new AbstractOneInputOneOutputOneFramePushRuntime() {
-
- private ArrayBackedValueStorage evalOutput;
- private IUnnestingFunction agg;
+ private IPointable p = VoidPointable.FACTORY.createPointable();
+ private IUnnestingEvaluator agg;
private ArrayTupleBuilder tupleBuilder;
@Override
public void open() throws HyracksDataException {
- initAccessAppendRef(context);
- evalOutput = new ArrayBackedValueStorage();
+ initAccessAppendRef(ctx);
try {
- agg = unnestingFactory.createUnnestingFunction(evalOutput);
+ agg = unnestingFactory.createUnnestingEvaluator(ctx);
} catch (AlgebricksException ae) {
throw new HyracksDataException(ae);
}
@@ -92,8 +91,7 @@
boolean goon = true;
do {
tupleBuilder.reset();
- evalOutput.reset();
- if (!agg.step()) {
+ if (!agg.step(p)) {
goon = false;
} else {
if (!outColIsProjected) {
@@ -102,8 +100,7 @@
for (int f = 0; f < outColPos; f++) {
tupleBuilder.addField(tAccess, t, f);
}
- tupleBuilder.addField(evalOutput.getBytes(), evalOutput.getStartIndex(), evalOutput
- .getLength());
+ tupleBuilder.addField(p.getByteArray(), p.getStartOffset(), p.getLength());
for (int f = outColPos + 1; f < projectionList.length; f++) {
tupleBuilder.addField(tAccess, t, f);
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/writers/PrinterBasedWriterFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/writers/PrinterBasedWriterFactory.java
similarity index 73%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/writers/PrinterBasedWriterFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/writers/PrinterBasedWriterFactory.java
index 36ef827..9c53241 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/writers/PrinterBasedWriterFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/writers/PrinterBasedWriterFactory.java
@@ -1,14 +1,14 @@
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.writers;
+package edu.uci.ics.hyracks.algebricks.runtime.writers;
import java.io.PrintStream;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IAWriter;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IAWriterFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinter;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IAWriter;
+import edu.uci.ics.hyracks.algebricks.data.IAWriterFactory;
+import edu.uci.ics.hyracks.algebricks.data.IPrinter;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
public class PrinterBasedWriterFactory implements IAWriterFactory {
@@ -37,7 +37,7 @@
}
@Override
- public void printTuple(FrameTupleAccessor tAccess, int tIdx) throws AlgebricksException {
+ public void printTuple(IFrameTupleAccessor tAccess, int tIdx) throws AlgebricksException {
for (int i = 0; i < fields.length; i++) {
int fldStart = tAccess.getTupleStartOffset(tIdx) + tAccess.getFieldSlotsLength()
+ tAccess.getFieldStartOffset(tIdx, fields[i]);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/writers/SerializedDataWriterFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/writers/SerializedDataWriterFactory.java
similarity index 74%
rename from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/writers/SerializedDataWriterFactory.java
rename to hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/writers/SerializedDataWriterFactory.java
index 79d7a99..61f822e 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/writers/SerializedDataWriterFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/writers/SerializedDataWriterFactory.java
@@ -1,16 +1,16 @@
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.writers;
+package edu.uci.ics.hyracks.algebricks.runtime.writers;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.io.PrintStream;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IAWriter;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IAWriterFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IAWriter;
+import edu.uci.ics.hyracks.algebricks.data.IAWriterFactory;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
public class SerializedDataWriterFactory implements IAWriterFactory {
@@ -36,7 +36,7 @@
}
@Override
- public void printTuple(FrameTupleAccessor tAccess, int tIdx) throws AlgebricksException {
+ public void printTuple(IFrameTupleAccessor tAccess, int tIdx) throws AlgebricksException {
for (int i = 0; i < fields.length; i++) {
int fldStart = tAccess.getTupleStartOffset(tIdx) + tAccess.getFieldSlotsLength()
+ tAccess.getFieldStartOffset(tIdx, fields[i]);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/pom.xml b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/pom.xml
index e700b0b..9afb096 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/pom.xml
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/pom.xml
@@ -20,19 +20,6 @@
</configuration>
</plugin>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.7.2</version>
- <configuration>
- <forkMode>pertest</forkMode>
- <argLine>-enableassertions -Djava.util.logging.config.file=src/test/resources/logging.properties</argLine>
- <includes>
- <include>**/*Test.java</include>
- <include>**/*Suite.java</include>
- </includes>
- </configuration>
- </plugin>
- <plugin>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntArrayUnnester.java b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntArrayUnnester.java
index ed98eec..bab293c 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntArrayUnnester.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntArrayUnnester.java
@@ -14,16 +14,17 @@
*/
package edu.uci.ics.hyracks.algebricks.tests.pushruntime;
-import java.io.DataOutput;
import java.io.IOException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IUnnestingFunction;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IUnnestingFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-public class IntArrayUnnester implements IUnnestingFunctionFactory {
+public class IntArrayUnnester implements IUnnestingEvaluatorFactory {
private int[] x;
@@ -34,11 +35,9 @@
private static final long serialVersionUID = 1L;
@Override
- public IUnnestingFunction createUnnestingFunction(IDataOutputProvider provider) throws AlgebricksException {
-
- final DataOutput out = provider.getDataOutput();
-
- return new IUnnestingFunction() {
+ public IUnnestingEvaluator createUnnestingEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
+ final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ return new IUnnestingEvaluator() {
private int pos;
@@ -48,12 +47,14 @@
}
@Override
- public boolean step() throws AlgebricksException {
+ public boolean step(IPointable result) throws AlgebricksException {
try {
if (pos < x.length) {
// Writes one byte to distinguish between null
// values and end of sequence.
- out.writeInt(x[pos]);
+ abvs.reset();
+ abvs.getDataOutput().writeInt(x[pos]);
+ result.set(abvs);
++pos;
return true;
} else {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerAddEvalFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerAddEvalFactory.java
index 57c0789..cdfac91 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerAddEvalFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerAddEvalFactory.java
@@ -14,50 +14,50 @@
*/
package edu.uci.ics.hyracks.algebricks.tests.pushruntime;
-import java.io.DataOutput;
import java.io.IOException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-public class IntegerAddEvalFactory implements IEvaluatorFactory {
+public class IntegerAddEvalFactory implements IScalarEvaluatorFactory {
private static final long serialVersionUID = 1L;
- private IEvaluatorFactory evalLeftFactory;
- private IEvaluatorFactory evalRightFactory;
+ private IScalarEvaluatorFactory evalLeftFactory;
+ private IScalarEvaluatorFactory evalRightFactory;
- public IntegerAddEvalFactory(IEvaluatorFactory evalLeftFactory, IEvaluatorFactory evalRightFactory) {
+ public IntegerAddEvalFactory(IScalarEvaluatorFactory evalLeftFactory, IScalarEvaluatorFactory evalRightFactory) {
this.evalLeftFactory = evalLeftFactory;
this.evalRightFactory = evalRightFactory;
}
@Override
- public IEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
- return new IEvaluator() {
-
- private DataOutput out = output.getDataOutput();
+ public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws AlgebricksException {
+ return new IScalarEvaluator() {
+ private IPointable p = VoidPointable.FACTORY.createPointable();
private ArrayBackedValueStorage argOut = new ArrayBackedValueStorage();
- private IEvaluator evalLeft = evalLeftFactory.createEvaluator(argOut);
- private IEvaluator evalRight = evalRightFactory.createEvaluator(argOut);
+ private IScalarEvaluator evalLeft = evalLeftFactory.createScalarEvaluator(ctx);
+ private IScalarEvaluator evalRight = evalRightFactory.createScalarEvaluator(ctx);
@SuppressWarnings("static-access")
@Override
- public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
- argOut.reset();
- evalLeft.evaluate(tuple);
- int v1 = IntegerSerializerDeserializer.INSTANCE.getInt(argOut.getBytes(), 0);
- argOut.reset();
- evalRight.evaluate(tuple);
- int v2 = IntegerSerializerDeserializer.INSTANCE.getInt(argOut.getBytes(), 0);
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws AlgebricksException {
+ evalLeft.evaluate(tuple, p);
+ int v1 = IntegerSerializerDeserializer.INSTANCE.getInt(p.getByteArray(), p.getStartOffset());
+ evalRight.evaluate(tuple, p);
+ int v2 = IntegerSerializerDeserializer.INSTANCE.getInt(p.getByteArray(), p.getStartOffset());
try {
- out.writeInt(v1 + v2);
+ argOut.reset();
+ argOut.getDataOutput().writeInt(v1 + v2);
+ result.set(argOut);
} catch (IOException e) {
throw new AlgebricksException(e);
}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerConstantEvalFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerConstantEvalFactory.java
index 880a4df..0eb9e88 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerConstantEvalFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerConstantEvalFactory.java
@@ -14,19 +14,17 @@
*/
package edu.uci.ics.hyracks.algebricks.tests.pushruntime;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-public class IntegerConstantEvalFactory implements IEvaluatorFactory {
+public class IntegerConstantEvalFactory implements IScalarEvaluatorFactory {
private static final long serialVersionUID = 1L;
private final int value;
@@ -41,31 +39,24 @@
}
@Override
- public IEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
- return new IEvaluator() {
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
+ return new IScalarEvaluator() {
- private DataOutput out = output.getDataOutput();
private ArrayBackedValueStorage buf = new ArrayBackedValueStorage();
- boolean first = true;
- @Override
- public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
- if (first) {
- first = false;
- try {
- IntegerSerializerDeserializer.INSTANCE.serialize(value, buf.getDataOutput());
- } catch (HyracksDataException e) {
- throw new AlgebricksException(e);
- }
- }
-
+ {
try {
- out.write(buf.getBytes(), 0, buf.getLength());
- } catch (IOException e) {
+ IntegerSerializerDeserializer.INSTANCE.serialize(value, buf.getDataOutput());
+ } catch (HyracksDataException e) {
throw new AlgebricksException(e);
}
}
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws AlgebricksException {
+ result.set(buf);
+ }
};
}
-}
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerEqualsEvalFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerEqualsEvalFactory.java
index e15851b..d3d2938 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerEqualsEvalFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerEqualsEvalFactory.java
@@ -14,52 +14,44 @@
*/
package edu.uci.ics.hyracks.algebricks.tests.pushruntime;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.primitive.BooleanPointable;
+import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-public class IntegerEqualsEvalFactory implements IEvaluatorFactory {
+public class IntegerEqualsEvalFactory implements IScalarEvaluatorFactory {
private static final long serialVersionUID = 1L;
- private IEvaluatorFactory evalFact1, evalFact2;
+ private IScalarEvaluatorFactory evalFact1, evalFact2;
- public IntegerEqualsEvalFactory(IEvaluatorFactory evalFact1, IEvaluatorFactory evalFact2) {
+ public IntegerEqualsEvalFactory(IScalarEvaluatorFactory evalFact1, IScalarEvaluatorFactory evalFact2) {
this.evalFact1 = evalFact1;
this.evalFact2 = evalFact2;
}
@Override
- public IEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
- return new IEvaluator() {
- private DataOutput dataout = output.getDataOutput();
- private ArrayBackedValueStorage out1 = new ArrayBackedValueStorage();
- private ArrayBackedValueStorage out2 = new ArrayBackedValueStorage();
- private IEvaluator eval1 = evalFact1.createEvaluator(out1);
- private IEvaluator eval2 = evalFact2.createEvaluator(out2);
+ public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws AlgebricksException {
+ return new IScalarEvaluator() {
+ private IPointable p = VoidPointable.FACTORY.createPointable();
+ private IScalarEvaluator eval1 = evalFact1.createScalarEvaluator(ctx);
+ private IScalarEvaluator eval2 = evalFact2.createScalarEvaluator(ctx);
+ private byte[] rBytes = new byte[1];
@Override
- public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
- out1.reset();
- eval1.evaluate(tuple);
- out2.reset();
- eval2.evaluate(tuple);
- int v1 = IntegerSerializerDeserializer.getInt(out1.getBytes(), 0);
- int v2 = IntegerSerializerDeserializer.getInt(out2.getBytes(), 0);
- boolean r = v1 == v2;
- try {
- dataout.writeBoolean(r);
- } catch (IOException ioe) {
- throw new AlgebricksException(ioe);
- }
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws AlgebricksException {
+ eval1.evaluate(tuple, p);
+ int v1 = IntegerSerializerDeserializer.getInt(p.getByteArray(), p.getStartOffset());
+ eval2.evaluate(tuple, p);
+ int v2 = IntegerSerializerDeserializer.getInt(p.getByteArray(), p.getStartOffset());
+ BooleanPointable.setBoolean(rBytes, 0, v1 == v2);
+ result.set(rBytes, 0, 1);
}
};
}
-}
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerGreaterThanEvalFactory.java b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerGreaterThanEvalFactory.java
index 5c5d6d4..8f37922 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerGreaterThanEvalFactory.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerGreaterThanEvalFactory.java
@@ -14,52 +14,44 @@
*/
package edu.uci.ics.hyracks.algebricks.tests.pushruntime;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.primitive.BooleanPointable;
+import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-public class IntegerGreaterThanEvalFactory implements IEvaluatorFactory {
+public class IntegerGreaterThanEvalFactory implements IScalarEvaluatorFactory {
private static final long serialVersionUID = 1L;
- private IEvaluatorFactory evalFact1, evalFact2;
+ private IScalarEvaluatorFactory evalFact1, evalFact2;
- public IntegerGreaterThanEvalFactory(IEvaluatorFactory evalFact1, IEvaluatorFactory evalFact2) {
+ public IntegerGreaterThanEvalFactory(IScalarEvaluatorFactory evalFact1, IScalarEvaluatorFactory evalFact2) {
this.evalFact1 = evalFact1;
this.evalFact2 = evalFact2;
}
@Override
- public IEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
- return new IEvaluator() {
- private DataOutput dataout = output.getDataOutput();
- private ArrayBackedValueStorage out1 = new ArrayBackedValueStorage();
- private ArrayBackedValueStorage out2 = new ArrayBackedValueStorage();
- private IEvaluator eval1 = evalFact1.createEvaluator(out1);
- private IEvaluator eval2 = evalFact2.createEvaluator(out2);
+ public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws AlgebricksException {
+ return new IScalarEvaluator() {
+ private IPointable p = VoidPointable.FACTORY.createPointable();
+ private IScalarEvaluator eval1 = evalFact1.createScalarEvaluator(ctx);
+ private IScalarEvaluator eval2 = evalFact2.createScalarEvaluator(ctx);
+ private byte[] rBytes = new byte[1];
@Override
- public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
- out1.reset();
- eval1.evaluate(tuple);
- out2.reset();
- eval2.evaluate(tuple);
- int v1 = IntegerSerializerDeserializer.getInt(out1.getBytes(), 0);
- int v2 = IntegerSerializerDeserializer.getInt(out2.getBytes(), 0);
- boolean r = v1 > v2;
- try {
- dataout.writeBoolean(r);
- } catch (IOException ioe) {
- throw new AlgebricksException(ioe);
- }
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws AlgebricksException {
+ eval1.evaluate(tuple, p);
+ int v1 = IntegerSerializerDeserializer.getInt(p.getByteArray(), p.getStartOffset());
+ eval2.evaluate(tuple, p);
+ int v2 = IntegerSerializerDeserializer.getInt(p.getByteArray(), p.getStartOffset());
+ BooleanPointable.setBoolean(rBytes, 0, v1 > v2);
+ result.set(rBytes, 0, 1);
}
};
}
-}
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
index 0370632..3de0966 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
@@ -11,40 +11,40 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.aggregators.TupleCountAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.aggregators.TupleCountRunningAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IRunningAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IUnnestingFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.evaluators.ColumnAccessEvalFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data.BinaryBooleanInspectorImpl;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data.BinaryIntegerInspectorImpl;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data.IntegerPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data.NoopNullWriterFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data.UTF8StringPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg.AggregateRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg.NestedPlansAccumulatingAggregatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg.SimpleAlgebricksAccumulatingAggregatorFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.group.MicroPreClusteredGroupRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta.SubplanRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.sort.InMemorySortRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.AssignRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.EmptyTupleSourceRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.NestedTupleSourceRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.PrinterRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.RunningAggregateRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.SinkWriterRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StreamLimitRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StreamProjectRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StreamSelectRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StringStreamingRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.UnnestRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.writers.PrinterBasedWriterFactory;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.data.impl.BinaryBooleanInspectorImpl;
+import edu.uci.ics.hyracks.algebricks.data.impl.BinaryIntegerInspectorImpl;
+import edu.uci.ics.hyracks.algebricks.data.impl.IntegerPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.data.impl.NoopNullWriterFactory;
+import edu.uci.ics.hyracks.algebricks.data.impl.UTF8StringPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.aggregators.TupleCountAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.aggregators.TupleCountRunningAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.evaluators.TupleFieldEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.AggregateRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.NestedPlansAccumulatingAggregatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.SimpleAlgebricksAccumulatingAggregatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.group.MicroPreClusteredGroupRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.meta.SubplanRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.sort.InMemorySortRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.AssignRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.EmptyTupleSourceRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.NestedTupleSourceRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.PrinterRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.RunningAggregateRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.SinkWriterRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StreamLimitRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StreamProjectRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StreamSelectRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StringStreamingRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.UnnestRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.writers.PrinterBasedWriterFactory;
import edu.uci.ics.hyracks.algebricks.tests.util.AlgebricksHyracksIntegrationUtil;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
@@ -75,9 +75,9 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.dataflow.std.file.LineFileWriteOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.group.HashGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.hash.HashGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.preclustered.PreclusteredGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.misc.SplitOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.InMemorySortOperatorDescriptor;
@@ -115,8 +115,8 @@
EmptyTupleSourceRuntimeFactory ets = new EmptyTupleSourceRuntimeFactory();
RecordDescriptor etsDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
- AssignRuntimeFactory assign = new AssignRuntimeFactory(new int[] { 0, 1 }, new IEvaluatorFactory[] { const1,
- const2 }, new int[] { 0, 1 });
+ AssignRuntimeFactory assign = new AssignRuntimeFactory(new int[] { 0, 1 }, new IScalarEvaluatorFactory[] {
+ const1, const2 }, new int[] { 0, 1 });
RecordDescriptor assignDesc = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
@@ -139,8 +139,8 @@
EmptyTupleSourceRuntimeFactory ets = new EmptyTupleSourceRuntimeFactory();
RecordDescriptor etsDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
- AssignRuntimeFactory assign = new AssignRuntimeFactory(new int[] { 0, 1 }, new IEvaluatorFactory[] { const1,
- const2 }, new int[] { 0, 1 });
+ AssignRuntimeFactory assign = new AssignRuntimeFactory(new int[] { 0, 1 }, new IScalarEvaluatorFactory[] {
+ const1, const2 }, new int[] { 0, 1 });
RecordDescriptor assignDesc = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
@@ -179,10 +179,10 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, intScanner, DEFAULT_NODES);
// the algebricks op.
- IEvaluatorFactory cond = new IntegerGreaterThanEvalFactory(new IntegerConstantEvalFactory(2),
- new ColumnAccessEvalFactory(0));
+ IScalarEvaluatorFactory cond = new IntegerGreaterThanEvalFactory(new IntegerConstantEvalFactory(2),
+ new TupleFieldEvaluatorFactory(0));
StreamSelectRuntimeFactory select = new StreamSelectRuntimeFactory(cond, new int[] { 0 },
- BinaryBooleanInspectorImpl.INSTANCE);
+ BinaryBooleanInspectorImpl.FACTORY);
RecordDescriptor selectDesc = intScannerDesc;
String filePath = PATH_ACTUAL + SEPARATOR + "scanSelectWrite.out";
@@ -216,8 +216,8 @@
EmptyTupleSourceRuntimeFactory ets = new EmptyTupleSourceRuntimeFactory();
RecordDescriptor etsDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
- AssignRuntimeFactory assign = new AssignRuntimeFactory(new int[] { 0, 1 }, new IEvaluatorFactory[] { const1,
- const2 }, new int[] { 0, 1 });
+ AssignRuntimeFactory assign = new AssignRuntimeFactory(new int[] { 0, 1 }, new IScalarEvaluatorFactory[] {
+ const1, const2 }, new int[] { 0, 1 });
RecordDescriptor assignDesc = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
StreamProjectRuntimeFactory project = new StreamProjectRuntimeFactory(new int[] { 1 });
@@ -270,7 +270,7 @@
// the algebricks op.
StreamLimitRuntimeFactory limit = new StreamLimitRuntimeFactory(new IntegerConstantEvalFactory(2), null,
- new int[] { 0 }, BinaryIntegerInspectorImpl.INSTANCE);
+ new int[] { 0 }, BinaryIntegerInspectorImpl.FACTORY);
RecordDescriptor limitDesc = new RecordDescriptor(
new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
@@ -302,7 +302,7 @@
EmptyTupleSourceRuntimeFactory ets = new EmptyTupleSourceRuntimeFactory();
RecordDescriptor etsDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
- IUnnestingFunctionFactory aggregFactory = new IntArrayUnnester(new int[] { 100, 200, 300 });
+ IUnnestingEvaluatorFactory aggregFactory = new IntArrayUnnester(new int[] { 100, 200, 300 });
UnnestRuntimeFactory unnest = new UnnestRuntimeFactory(0, aggregFactory, new int[] { 0 });
RecordDescriptor unnestDesc = new RecordDescriptor(
new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
@@ -352,7 +352,7 @@
// the algebricks op.
AggregateRuntimeFactory agg = new AggregateRuntimeFactory(
- new IAggregateFunctionFactory[] { new TupleCountAggregateFunctionFactory() });
+ new IAggregateEvaluatorFactory[] { new TupleCountAggregateFunctionFactory() });
RecordDescriptor aggDesc = new RecordDescriptor(
new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
@@ -415,7 +415,7 @@
NestedTupleSourceRuntimeFactory nts = new NestedTupleSourceRuntimeFactory();
RecordDescriptor ntsDesc = sortDesc;
AggregateRuntimeFactory agg = new AggregateRuntimeFactory(
- new IAggregateFunctionFactory[] { new TupleCountAggregateFunctionFactory() });
+ new IAggregateEvaluatorFactory[] { new TupleCountAggregateFunctionFactory() });
RecordDescriptor aggDesc = new RecordDescriptor(
new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
AlgebricksPipeline pipeline = new AlgebricksPipeline(new IPushRuntimeFactory[] { nts, agg },
@@ -432,10 +432,10 @@
new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
// the algebricks op.
- IEvaluatorFactory cond = new IntegerEqualsEvalFactory(new IntegerConstantEvalFactory(3),
- new ColumnAccessEvalFactory(0)); // Canadian customers
+ IScalarEvaluatorFactory cond = new IntegerEqualsEvalFactory(new IntegerConstantEvalFactory(3),
+ new TupleFieldEvaluatorFactory(0)); // Canadian customers
StreamSelectRuntimeFactory select = new StreamSelectRuntimeFactory(cond, new int[] { 1 },
- BinaryBooleanInspectorImpl.INSTANCE);
+ BinaryBooleanInspectorImpl.FACTORY);
RecordDescriptor selectDesc = new RecordDescriptor(
new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
@@ -491,7 +491,7 @@
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(new int[] { 3 },
new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY) });
- IAggregateFunctionFactory[] aggFuns = new IAggregateFunctionFactory[] { new TupleCountAggregateFunctionFactory() };
+ IAggregateEvaluatorFactory[] aggFuns = new IAggregateEvaluatorFactory[] { new TupleCountAggregateFunctionFactory() };
IAggregatorDescriptorFactory aggFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(aggFuns,
new int[] { 3 }, new int[] {});
HashGroupOperatorDescriptor gby = new HashGroupOperatorDescriptor(spec, new int[] { 3 }, tpcf,
@@ -502,10 +502,10 @@
new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
// the algebricks op.
- IEvaluatorFactory cond = new IntegerEqualsEvalFactory(new IntegerConstantEvalFactory(3),
- new ColumnAccessEvalFactory(0)); // Canadian customers
+ IScalarEvaluatorFactory cond = new IntegerEqualsEvalFactory(new IntegerConstantEvalFactory(3),
+ new TupleFieldEvaluatorFactory(0)); // Canadian customers
StreamSelectRuntimeFactory select = new StreamSelectRuntimeFactory(cond, new int[] { 1 },
- BinaryBooleanInspectorImpl.INSTANCE);
+ BinaryBooleanInspectorImpl.FACTORY);
RecordDescriptor selectDesc = new RecordDescriptor(
new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
@@ -538,13 +538,13 @@
EmptyTupleSourceRuntimeFactory ets = new EmptyTupleSourceRuntimeFactory();
RecordDescriptor etsDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
- IUnnestingFunctionFactory aggregFactory = new IntArrayUnnester(new int[] { 100, 200, 300 });
+ IUnnestingEvaluatorFactory aggregFactory = new IntArrayUnnester(new int[] { 100, 200, 300 });
UnnestRuntimeFactory unnest = new UnnestRuntimeFactory(0, aggregFactory, new int[] { 0 });
RecordDescriptor unnestDesc = new RecordDescriptor(
new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
RunningAggregateRuntimeFactory ragg = new RunningAggregateRuntimeFactory(new int[] { 1 },
- new IRunningAggregateFunctionFactory[] { new TupleCountRunningAggregateFunctionFactory() }, new int[] {
+ new IRunningAggregateEvaluatorFactory[] { new TupleCountRunningAggregateFunctionFactory() }, new int[] {
0, 1 });
RecordDescriptor raggDesc = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
@@ -579,8 +579,8 @@
EmptyTupleSourceRuntimeFactory ets = new EmptyTupleSourceRuntimeFactory();
RecordDescriptor etsDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
- AssignRuntimeFactory assign = new AssignRuntimeFactory(new int[] { 0, 1 }, new IEvaluatorFactory[] { const1,
- const2 }, new int[] { 0, 1 });
+ AssignRuntimeFactory assign = new AssignRuntimeFactory(new int[] { 0, 1 }, new IScalarEvaluatorFactory[] {
+ const1, const2 }, new int[] { 0, 1 });
RecordDescriptor assignDesc = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
@@ -737,15 +737,15 @@
EmptyTupleSourceRuntimeFactory ets = new EmptyTupleSourceRuntimeFactory();
RecordDescriptor etsDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
- AssignRuntimeFactory assign1 = new AssignRuntimeFactory(new int[] { 0 }, new IEvaluatorFactory[] { const1 },
- new int[] { 0 });
+ AssignRuntimeFactory assign1 = new AssignRuntimeFactory(new int[] { 0 },
+ new IScalarEvaluatorFactory[] { const1 }, new int[] { 0 });
RecordDescriptor assign1Desc = new RecordDescriptor(
new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
NestedTupleSourceRuntimeFactory nts = new NestedTupleSourceRuntimeFactory();
AssignRuntimeFactory assign2 = new AssignRuntimeFactory(new int[] { 1 },
- new IEvaluatorFactory[] { new IntegerAddEvalFactory(new ColumnAccessEvalFactory(0), const2) },
+ new IScalarEvaluatorFactory[] { new IntegerAddEvalFactory(new TupleFieldEvaluatorFactory(0), const2) },
new int[] { 0, 1 });
RecordDescriptor assign2Desc = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
@@ -821,7 +821,7 @@
NestedTupleSourceRuntimeFactory nts = new NestedTupleSourceRuntimeFactory();
RecordDescriptor ntsDesc = sortDesc;
AggregateRuntimeFactory agg = new AggregateRuntimeFactory(
- new IAggregateFunctionFactory[] { new TupleCountAggregateFunctionFactory() });
+ new IAggregateEvaluatorFactory[] { new TupleCountAggregateFunctionFactory() });
RecordDescriptor aggDesc = new RecordDescriptor(
new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
AlgebricksPipeline pipeline = new AlgebricksPipeline(new IPushRuntimeFactory[] { nts, agg },
@@ -835,10 +835,10 @@
npaaf, sortDesc, gbyDesc, null);
// the algebricks op.
- IEvaluatorFactory cond = new IntegerEqualsEvalFactory(new IntegerConstantEvalFactory(3),
- new ColumnAccessEvalFactory(0)); // Canadian customers
+ IScalarEvaluatorFactory cond = new IntegerEqualsEvalFactory(new IntegerConstantEvalFactory(3),
+ new TupleFieldEvaluatorFactory(0)); // Canadian customers
StreamSelectRuntimeFactory select = new StreamSelectRuntimeFactory(cond, new int[] { 1 },
- BinaryBooleanInspectorImpl.INSTANCE);
+ BinaryBooleanInspectorImpl.FACTORY);
RecordDescriptor selectDesc = new RecordDescriptor(
new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/tools/WriteValueTest.java b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/tools/WriteValueTest.java
index 18f0e75..25f2dc0 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/tools/WriteValueTest.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/tools/WriteValueTest.java
@@ -5,8 +5,8 @@
import org.junit.Test;
-import edu.uci.ics.hyracks.algebricks.core.utils.WriteValueTools;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ByteArrayAccessibleOutputStream;
+import edu.uci.ics.hyracks.algebricks.data.utils.WriteValueTools;
+import edu.uci.ics.hyracks.data.std.util.ByteArrayAccessibleOutputStream;
public class WriteValueTest {
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java
index 922b0fd..b5c4b47 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java
+++ b/hyracks/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java
@@ -79,9 +79,8 @@
}
public static void runJob(JobSpecification spec) throws Exception {
- JobId jobId = hcc.createJob(AlgebricksConfig.HYRACKS_APP_NAME, spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
AlgebricksConfig.ALGEBRICKS_LOGGER.info(spec.toJSON().toString());
- hcc.start(jobId);
+ JobId jobId = hcc.startJob(AlgebricksConfig.HYRACKS_APP_NAME, spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
AlgebricksConfig.ALGEBRICKS_LOGGER.info(jobId.toString());
hcc.waitForCompletion(jobId);
}
diff --git a/hyracks/hyracks-algebricks/pom.xml b/hyracks/hyracks-algebricks/pom.xml
index 1e17864..962b6a1 100644
--- a/hyracks/hyracks-algebricks/pom.xml
+++ b/hyracks/hyracks-algebricks/pom.xml
@@ -11,7 +11,10 @@
<modules>
<module>hyracks-algebricks-compiler</module>
+ <module>hyracks-algebricks-common</module>
+ <module>hyracks-algebricks-data</module>
<module>hyracks-algebricks-core</module>
+ <module>hyracks-algebricks-runtime</module>
<module>hyracks-algebricks-rewriter</module>
<module>hyracks-algebricks-tests</module>
<module>hyracks-algebricks-examples</module>
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/IApplicationContext.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/IApplicationContext.java
index c07de41..81cf511 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/IApplicationContext.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/IApplicationContext.java
@@ -16,6 +16,8 @@
import java.io.Serializable;
+import edu.uci.ics.hyracks.api.messages.IMessageBroker;
+
/**
* Base class of the {@link ICCApplicationContext} and the
* {@link INCApplicationContext}.
@@ -39,4 +41,10 @@
* @return
*/
public Serializable getDistributedState();
+
+ public void setMessageBroker(IMessageBroker messageBroker);
+
+ public IMessageBroker getMessageBroker();
+
+ public String getApplicationName();
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/ICCApplicationContext.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/ICCApplicationContext.java
index 2e02101..2792d29 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/ICCApplicationContext.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/ICCApplicationContext.java
@@ -18,21 +18,16 @@
import edu.uci.ics.hyracks.api.context.ICCContext;
import edu.uci.ics.hyracks.api.job.IJobLifecycleListener;
-import edu.uci.ics.hyracks.api.job.IJobSpecificationFactory;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
/**
* Application Context at the Cluster Controller for an application.
*
* @author vinayakb
- *
*/
public interface ICCApplicationContext extends IApplicationContext {
/**
* Sets the state that must be distributed by the infrastructure to all the
- * NC application contects. Any state set by calling thsi method in the
- * {@link ICCBootstrap#start()} call is made available to all the
- * {@link INCApplicationContext} objects at each Node Controller. The state
+ * NC application contects. Any state set by calling thsi method in the {@link ICCBootstrap#start()} call is made available to all the {@link INCApplicationContext} objects at each Node Controller. The state
* is then available to be inspected by the application at the NC during or
* after the {@link INCBootstrap#start()} call.
*
@@ -42,17 +37,6 @@
public void setDistributedState(Serializable state);
/**
- * A factory class specific to this application that may accept incoming
- * {@link JobSpecification} and produce modified {@link JobSpecification}
- * that is executed on the cluster. If a {@link IJobSpecificationFactory} is
- * not set, the incoming {@link JobSpecification} is executed unmodified.
- *
- * @param jobSpecFactory
- * - The Job Specification Factory.
- */
- public void setJobSpecificationFactory(IJobSpecificationFactory jobSpecFactory);
-
- /**
* A listener that listens to Job Lifecycle events at the Cluster
* Controller.
*
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/channels/IInputChannel.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/channels/IInputChannel.java
index f79a464..a8f2fda 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/channels/IInputChannel.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/channels/IInputChannel.java
@@ -16,6 +16,7 @@
import java.nio.ByteBuffer;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
public interface IInputChannel {
@@ -29,7 +30,7 @@
public void recycleBuffer(ByteBuffer buffer);
- public void open() throws HyracksDataException;
+ public void open(IHyracksTaskContext ctx) throws HyracksDataException;
public void close() throws HyracksDataException;
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceFunctions.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceFunctions.java
index a7f324d..e34e60d 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceFunctions.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceFunctions.java
@@ -23,6 +23,7 @@
public class HyracksClientInterfaceFunctions {
public enum FunctionId {
GET_CLUSTER_CONTROLLER_INFO,
+ GET_CLUSTER_TOPOLOGY,
CREATE_APPLICATION,
START_APPLICATION,
DESTROY_APPLICATION,
@@ -105,37 +106,6 @@
}
}
- public static class CreateJobFunction extends Function {
- private static final long serialVersionUID = 1L;
-
- private final String appName;
- private final byte[] jobSpec;
- private final EnumSet<JobFlag> jobFlags;
-
- public CreateJobFunction(String appName, byte[] jobSpec, EnumSet<JobFlag> jobFlags) {
- this.appName = appName;
- this.jobSpec = jobSpec;
- this.jobFlags = jobFlags;
- }
-
- @Override
- public FunctionId getFunctionId() {
- return FunctionId.CREATE_JOB;
- }
-
- public String getAppName() {
- return appName;
- }
-
- public byte[] getJobSpec() {
- return jobSpec;
- }
-
- public EnumSet<JobFlag> getJobFlags() {
- return jobFlags;
- }
- }
-
public static class GetJobStatusFunction extends Function {
private static final long serialVersionUID = 1L;
@@ -158,10 +128,14 @@
public static class StartJobFunction extends Function {
private static final long serialVersionUID = 1L;
- private final JobId jobId;
+ private final String appName;
+ private final byte[] acggfBytes;
+ private final EnumSet<JobFlag> jobFlags;
- public StartJobFunction(JobId jobId) {
- this.jobId = jobId;
+ public StartJobFunction(String appName, byte[] acggfBytes, EnumSet<JobFlag> jobFlags) {
+ this.appName = appName;
+ this.acggfBytes = acggfBytes;
+ this.jobFlags = jobFlags;
}
@Override
@@ -169,8 +143,16 @@
return FunctionId.START_JOB;
}
- public JobId getJobId() {
- return jobId;
+ public String getAppName() {
+ return appName;
+ }
+
+ public byte[] getACGGFBytes() {
+ return acggfBytes;
+ }
+
+ public EnumSet<JobFlag> getJobFlags() {
+ return jobFlags;
}
}
@@ -201,4 +183,13 @@
return FunctionId.GET_NODE_CONTROLLERS_INFO;
}
}
+
+ public static class GetClusterTopologyFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.GET_CLUSTER_TOPOLOGY;
+ }
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceRemoteProxy.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceRemoteProxy.java
index 602288c..4c06d42 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceRemoteProxy.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceRemoteProxy.java
@@ -20,6 +20,7 @@
import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.api.topology.ClusterTopology;
import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
import edu.uci.ics.hyracks.ipc.api.RPCInterface;
@@ -61,13 +62,6 @@
}
@Override
- public JobId createJob(String appName, byte[] jobSpec, EnumSet<JobFlag> jobFlags) throws Exception {
- HyracksClientInterfaceFunctions.CreateJobFunction cjf = new HyracksClientInterfaceFunctions.CreateJobFunction(
- appName, jobSpec, jobFlags);
- return (JobId) rpci.call(ipcHandle, cjf);
- }
-
- @Override
public JobStatus getJobStatus(JobId jobId) throws Exception {
HyracksClientInterfaceFunctions.GetJobStatusFunction gjsf = new HyracksClientInterfaceFunctions.GetJobStatusFunction(
jobId);
@@ -75,10 +69,10 @@
}
@Override
- public void startJob(JobId jobId) throws Exception {
+ public JobId startJob(String appName, byte[] acggfBytes, EnumSet<JobFlag> jobFlags) throws Exception {
HyracksClientInterfaceFunctions.StartJobFunction sjf = new HyracksClientInterfaceFunctions.StartJobFunction(
- jobId);
- rpci.call(ipcHandle, sjf);
+ appName, acggfBytes, jobFlags);
+ return (JobId) rpci.call(ipcHandle, sjf);
}
@Override
@@ -93,4 +87,10 @@
HyracksClientInterfaceFunctions.GetNodeControllersInfoFunction gncif = new HyracksClientInterfaceFunctions.GetNodeControllersInfoFunction();
return (Map<String, NodeControllerInfo>) rpci.call(ipcHandle, gncif);
}
+
+ @Override
+ public ClusterTopology getClusterTopology() throws Exception {
+ HyracksClientInterfaceFunctions.GetClusterTopologyFunction gctf = new HyracksClientInterfaceFunctions.GetClusterTopologyFunction();
+ return (ClusterTopology) rpci.call(ipcHandle, gctf);
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksConnection.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksConnection.java
index 8c9409c..227524c 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksConnection.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksConnection.java
@@ -25,11 +25,14 @@
import org.apache.http.entity.FileEntity;
import org.apache.http.impl.client.DefaultHttpClient;
+import edu.uci.ics.hyracks.api.client.impl.JobSpecificationActivityClusterGraphGeneratorFactory;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.job.IActivityClusterGraphGeneratorFactory;
import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.api.topology.ClusterTopology;
import edu.uci.ics.hyracks.api.util.JavaSerializationUtils;
import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
import edu.uci.ics.hyracks.ipc.api.RPCInterface;
@@ -41,7 +44,6 @@
* Controller.
*
* @author vinayakb
- *
*/
public final class HyracksConnection implements IHyracksClientConnection {
private final String ccHost;
@@ -95,23 +97,25 @@
}
@Override
- public JobId createJob(String appName, JobSpecification jobSpec) throws Exception {
- return createJob(appName, jobSpec, EnumSet.noneOf(JobFlag.class));
- }
-
- @Override
- public JobId createJob(String appName, JobSpecification jobSpec, EnumSet<JobFlag> jobFlags) throws Exception {
- return hci.createJob(appName, JavaSerializationUtils.serialize(jobSpec), jobFlags);
- }
-
- @Override
public JobStatus getJobStatus(JobId jobId) throws Exception {
return hci.getJobStatus(jobId);
}
@Override
- public void start(JobId jobId) throws Exception {
- hci.startJob(jobId);
+ public JobId startJob(String appName, JobSpecification jobSpec) throws Exception {
+ return startJob(appName, jobSpec, EnumSet.noneOf(JobFlag.class));
+ }
+
+ @Override
+ public JobId startJob(String appName, JobSpecification jobSpec, EnumSet<JobFlag> jobFlags) throws Exception {
+ JobSpecificationActivityClusterGraphGeneratorFactory jsacggf = new JobSpecificationActivityClusterGraphGeneratorFactory(
+ jobSpec);
+ return startJob(appName, jsacggf, jobFlags);
+ }
+
+ public JobId startJob(String appName, IActivityClusterGraphGeneratorFactory acggf, EnumSet<JobFlag> jobFlags)
+ throws Exception {
+ return hci.startJob(appName, JavaSerializationUtils.serialize(acggf), jobFlags);
}
@Override
@@ -123,4 +127,9 @@
public Map<String, NodeControllerInfo> getNodeControllerInfos() throws Exception {
return hci.getNodeControllersInfo();
}
+
+ @Override
+ public ClusterTopology getClusterTopology() throws Exception {
+ return hci.getClusterTopology();
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientConnection.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientConnection.java
index 37a746b..bdbb544 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientConnection.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientConnection.java
@@ -18,16 +18,17 @@
import java.util.EnumSet;
import java.util.Map;
+import edu.uci.ics.hyracks.api.job.IActivityClusterGraphGeneratorFactory;
import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.api.topology.ClusterTopology;
/**
* Interface used by clients to communicate with the Hyracks Cluster Controller.
*
* @author vinayakb
- *
*/
public interface IHyracksClientConnection {
/**
@@ -51,35 +52,6 @@
public void destroyApplication(String appName) throws Exception;
/**
- * Creates a Job Instance in the specified Hyracks application using the
- * specified {@link JobSpecification}.
- *
- * @param appName
- * Name of the application
- * @param jobSpec
- * Job Specification
- * @return
- * @throws Exception
- */
- public JobId createJob(String appName, JobSpecification jobSpec) throws Exception;
-
- /**
- * Creates a Job Instance in the specified Hyracks application using the
- * specified {@link JobSpecification}. The specified flags are used to
- * configure the Job creation process.
- *
- * @param appName
- * Name of the application
- * @param jobSpec
- * Job Specification
- * @param jobFlags
- * Flags
- * @return
- * @throws Exception
- */
- public JobId createJob(String appName, JobSpecification jobSpec, EnumSet<JobFlag> jobFlags) throws Exception;
-
- /**
* Gets the status of the specified Job.
*
* @param jobId
@@ -92,11 +64,40 @@
/**
* Start the specified Job.
*
- * @param jobId
- * JobId of the Job.
+ * @param appName
+ * Name of the application
+ * @param jobSpec
+ * Job Specification
* @throws Exception
*/
- public void start(JobId jobId) throws Exception;
+ public JobId startJob(String appName, JobSpecification jobSpec) throws Exception;
+
+ /**
+ * Start the specified Job.
+ *
+ * @param appName
+ * Name of the application
+ * @param jobSpec
+ * Job Specification
+ * @param jobFlags
+ * Flags
+ * @throws Exception
+ */
+ public JobId startJob(String appName, JobSpecification jobSpec, EnumSet<JobFlag> jobFlags) throws Exception;
+
+ /**
+ * Start the specified Job.
+ *
+ * @param appName
+ * Name of the application
+ * @param acggf
+ * Activity Cluster Graph Generator Factory
+ * @param jobFlags
+ * Flags
+ * @throws Exception
+ */
+ public JobId startJob(String appName, IActivityClusterGraphGeneratorFactory acggf, EnumSet<JobFlag> jobFlags)
+ throws Exception;
/**
* Waits until the specified job has completed, either successfully or has
@@ -114,4 +115,12 @@
* @return Map of node name to node information.
*/
public Map<String, NodeControllerInfo> getNodeControllerInfos() throws Exception;
+
+ /**
+ * Get the cluster topology
+ *
+ * @return the cluster topology
+ * @throws Exception
+ */
+ public ClusterTopology getClusterTopology() throws Exception;
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientInterface.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientInterface.java
index 866d307..ef5906e 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientInterface.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientInterface.java
@@ -20,6 +20,7 @@
import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.api.topology.ClusterTopology;
public interface IHyracksClientInterface {
public ClusterControllerInfo getClusterControllerInfo() throws Exception;
@@ -30,13 +31,13 @@
public void destroyApplication(String appName) throws Exception;
- public JobId createJob(String appName, byte[] jobSpec, EnumSet<JobFlag> jobFlags) throws Exception;
-
public JobStatus getJobStatus(JobId jobId) throws Exception;
- public void startJob(JobId jobId) throws Exception;
+ public JobId startJob(String appName, byte[] acggfBytes, EnumSet<JobFlag> jobFlags) throws Exception;
public void waitForCompletion(JobId jobId) throws Exception;
public Map<String, NodeControllerInfo> getNodeControllersInfo() throws Exception;
+
+ public ClusterTopology getClusterTopology() throws Exception;
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/ActivityClusterGraphBuilder.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/ActivityClusterGraphBuilder.java
new file mode 100644
index 0000000..6c6c211
--- /dev/null
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/ActivityClusterGraphBuilder.java
@@ -0,0 +1,176 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.client.impl;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.json.JSONException;
+
+import edu.uci.ics.hyracks.api.dataflow.ActivityId;
+import edu.uci.ics.hyracks.api.dataflow.IActivity;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.job.ActivityCluster;
+import edu.uci.ics.hyracks.api.job.ActivityClusterGraph;
+import edu.uci.ics.hyracks.api.job.ActivityClusterId;
+import edu.uci.ics.hyracks.api.job.JobActivityGraph;
+import edu.uci.ics.hyracks.api.job.JobId;
+
+public class ActivityClusterGraphBuilder {
+ private static final Logger LOGGER = Logger.getLogger(ActivityClusterGraphBuilder.class.getName());
+
+ public ActivityClusterGraphBuilder() {
+ }
+
+ private static Pair<ActivityId, ActivityId> findMergePair(JobActivityGraph jag, Set<Set<ActivityId>> eqSets) {
+ for (Set<ActivityId> eqSet : eqSets) {
+ for (ActivityId t : eqSet) {
+ List<IConnectorDescriptor> inputList = jag.getActivityInputMap().get(t);
+ if (inputList != null) {
+ for (IConnectorDescriptor conn : inputList) {
+ ActivityId inTask = jag.getProducerActivity(conn.getConnectorId());
+ if (!eqSet.contains(inTask)) {
+ return Pair.<ActivityId, ActivityId> of(t, inTask);
+ }
+ }
+ }
+ List<IConnectorDescriptor> outputList = jag.getActivityOutputMap().get(t);
+ if (outputList != null) {
+ for (IConnectorDescriptor conn : outputList) {
+ ActivityId outTask = jag.getConsumerActivity(conn.getConnectorId());
+ if (!eqSet.contains(outTask)) {
+ return Pair.<ActivityId, ActivityId> of(t, outTask);
+ }
+ }
+ }
+ }
+ }
+ return null;
+ }
+
+ public ActivityClusterGraph inferActivityClusters(JobId jobId, JobActivityGraph jag) {
+ /*
+ * Build initial equivalence sets map. We create a map such that for each IOperatorTask, t -> { t }
+ */
+ Map<ActivityId, Set<ActivityId>> stageMap = new HashMap<ActivityId, Set<ActivityId>>();
+ Set<Set<ActivityId>> stages = new HashSet<Set<ActivityId>>();
+ for (ActivityId taskId : jag.getActivityMap().keySet()) {
+ Set<ActivityId> eqSet = new HashSet<ActivityId>();
+ eqSet.add(taskId);
+ stageMap.put(taskId, eqSet);
+ stages.add(eqSet);
+ }
+
+ boolean changed = true;
+ while (changed) {
+ changed = false;
+ Pair<ActivityId, ActivityId> pair = findMergePair(jag, stages);
+ if (pair != null) {
+ merge(stageMap, stages, pair.getLeft(), pair.getRight());
+ changed = true;
+ }
+ }
+
+ ActivityClusterGraph acg = new ActivityClusterGraph();
+ Map<ActivityId, ActivityCluster> acMap = new HashMap<ActivityId, ActivityCluster>();
+ int acCounter = 0;
+ Map<ActivityId, IActivity> activityNodeMap = jag.getActivityMap();
+ List<ActivityCluster> acList = new ArrayList<ActivityCluster>();
+ for (Set<ActivityId> stage : stages) {
+ ActivityCluster ac = new ActivityCluster(acg, new ActivityClusterId(jobId, acCounter++));
+ acList.add(ac);
+ for (ActivityId aid : stage) {
+ IActivity activity = activityNodeMap.get(aid);
+ ac.addActivity(activity);
+ acMap.put(aid, ac);
+ }
+ }
+
+ for (Set<ActivityId> stage : stages) {
+ for (ActivityId aid : stage) {
+ IActivity activity = activityNodeMap.get(aid);
+ ActivityCluster ac = acMap.get(aid);
+ List<IConnectorDescriptor> aOutputs = jag.getActivityOutputMap().get(aid);
+ if (aOutputs == null || aOutputs.isEmpty()) {
+ ac.addRoot(activity);
+ } else {
+ int nActivityOutputs = aOutputs.size();
+ for (int i = 0; i < nActivityOutputs; ++i) {
+ IConnectorDescriptor conn = aOutputs.get(i);
+ ac.addConnector(conn);
+ Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>> pcPair = jag.getConnectorActivityMap()
+ .get(conn.getConnectorId());
+ ac.connect(conn, activity, i, pcPair.getRight().getLeft(), pcPair.getRight().getRight(), jag
+ .getConnectorRecordDescriptorMap().get(conn.getConnectorId()));
+ }
+ }
+ }
+ }
+
+ Map<ActivityId, Set<ActivityId>> blocked2BlockerMap = jag.getBlocked2BlockerMap();
+ for (ActivityCluster s : acList) {
+ Map<ActivityId, Set<ActivityId>> acBlocked2BlockerMap = s.getBlocked2BlockerMap();
+ Set<ActivityCluster> blockerStages = new HashSet<ActivityCluster>();
+ for (ActivityId t : s.getActivityMap().keySet()) {
+ Set<ActivityId> blockerTasks = blocked2BlockerMap.get(t);
+ acBlocked2BlockerMap.put(t, blockerTasks);
+ if (blockerTasks != null) {
+ for (ActivityId bt : blockerTasks) {
+ blockerStages.add(acMap.get(bt));
+ }
+ }
+ }
+ for (ActivityCluster bs : blockerStages) {
+ s.getDependencies().add(bs);
+ }
+ }
+ acg.addActivityClusters(acList);
+
+ if (LOGGER.isLoggable(Level.FINE)) {
+ try {
+ LOGGER.fine(acg.toJSON().toString(2));
+ } catch (JSONException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+ return acg;
+ }
+
+ private void merge(Map<ActivityId, Set<ActivityId>> eqSetMap, Set<Set<ActivityId>> eqSets, ActivityId t1,
+ ActivityId t2) {
+ Set<ActivityId> stage1 = eqSetMap.get(t1);
+ Set<ActivityId> stage2 = eqSetMap.get(t2);
+
+ Set<ActivityId> mergedSet = new HashSet<ActivityId>();
+ mergedSet.addAll(stage1);
+ mergedSet.addAll(stage2);
+
+ eqSets.remove(stage1);
+ eqSets.remove(stage2);
+ eqSets.add(mergedSet);
+
+ for (ActivityId t : mergedSet) {
+ eqSetMap.put(t, mergedSet);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/IConnectorDescriptorVisitor.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/IConnectorDescriptorVisitor.java
similarity index 94%
rename from hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/IConnectorDescriptorVisitor.java
rename to hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/IConnectorDescriptorVisitor.java
index 1c8446f..5b508b3 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/IConnectorDescriptorVisitor.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/IConnectorDescriptorVisitor.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.control.cc.job;
+package edu.uci.ics.hyracks.api.client.impl;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/IOperatorDescriptorVisitor.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/IOperatorDescriptorVisitor.java
similarity index 94%
rename from hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/IOperatorDescriptorVisitor.java
rename to hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/IOperatorDescriptorVisitor.java
index 840faef..fcdf9f0 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/IOperatorDescriptorVisitor.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/IOperatorDescriptorVisitor.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.control.cc.job;
+package edu.uci.ics.hyracks.api.client.impl;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/JobActivityGraphBuilder.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/JobActivityGraphBuilder.java
new file mode 100644
index 0000000..b4b9b3c
--- /dev/null
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/JobActivityGraphBuilder.java
@@ -0,0 +1,127 @@
+package edu.uci.ics.hyracks.api.client.impl;
+
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.lang3.tuple.Pair;
+
+import edu.uci.ics.hyracks.api.dataflow.ActivityId;
+import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.IActivity;
+import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.job.JobActivityGraph;
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class JobActivityGraphBuilder implements IActivityGraphBuilder {
+ private static final Logger LOGGER = Logger.getLogger(JobActivityGraphBuilder.class.getName());
+
+ private final Map<ActivityId, IOperatorDescriptor> activityOperatorMap;
+
+ private final JobActivityGraph jag;
+
+ private final JobSpecification jobSpec;
+
+ private final Map<ConnectorDescriptorId, Pair<IActivity, Integer>> connectorProducerMap;
+
+ private final Map<ConnectorDescriptorId, Pair<IActivity, Integer>> connectorConsumerMap;
+
+ public JobActivityGraphBuilder(JobSpecification jobSpec, EnumSet<JobFlag> jobFlags) {
+ activityOperatorMap = new HashMap<ActivityId, IOperatorDescriptor>();
+ jag = new JobActivityGraph();
+ this.jobSpec = jobSpec;
+ connectorProducerMap = new HashMap<ConnectorDescriptorId, Pair<IActivity, Integer>>();
+ connectorConsumerMap = new HashMap<ConnectorDescriptorId, Pair<IActivity, Integer>>();
+ }
+
+ public void addConnector(IConnectorDescriptor conn) {
+ jag.getConnectorMap().put(conn.getConnectorId(), conn);
+ jag.getConnectorRecordDescriptorMap().put(conn.getConnectorId(), jobSpec.getConnectorRecordDescriptor(conn));
+ }
+
+ @Override
+ public void addBlockingEdge(IActivity blocker, IActivity blocked) {
+ addToValueSet(jag.getBlocked2BlockerMap(), blocked.getActivityId(), blocker.getActivityId());
+ }
+
+ @Override
+ public void addSourceEdge(int operatorInputIndex, IActivity task, int taskInputIndex) {
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest("Adding source edge: " + task.getActivityId() + ":" + operatorInputIndex + " -> "
+ + task.getActivityId() + ":" + taskInputIndex);
+ }
+ IOperatorDescriptor op = activityOperatorMap.get(task.getActivityId());
+ IConnectorDescriptor conn = jobSpec.getInputConnectorDescriptor(op, operatorInputIndex);
+ insertIntoIndexedMap(jag.getActivityInputMap(), task.getActivityId(), taskInputIndex, conn);
+ connectorConsumerMap.put(conn.getConnectorId(), Pair.of(task, taskInputIndex));
+ }
+
+ @Override
+ public void addTargetEdge(int operatorOutputIndex, IActivity task, int taskOutputIndex) {
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest("Adding target edge: " + task.getActivityId() + ":" + operatorOutputIndex + " -> "
+ + task.getActivityId() + ":" + taskOutputIndex);
+ }
+ IOperatorDescriptor op = activityOperatorMap.get(task.getActivityId());
+ IConnectorDescriptor conn = jobSpec.getOutputConnectorDescriptor(op, operatorOutputIndex);
+ insertIntoIndexedMap(jag.getActivityOutputMap(), task.getActivityId(), taskOutputIndex, conn);
+ connectorProducerMap.put(conn.getConnectorId(), Pair.of(task, taskOutputIndex));
+ }
+
+ @Override
+ public void addActivity(IOperatorDescriptor op, IActivity task) {
+ activityOperatorMap.put(task.getActivityId(), op);
+ ActivityId activityId = task.getActivityId();
+ jag.getActivityMap().put(activityId, task);
+ }
+
+ public void finish() {
+ Map<ConnectorDescriptorId, Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>>> caMap = jag
+ .getConnectorActivityMap();
+ for (Map.Entry<ConnectorDescriptorId, Pair<IActivity, Integer>> e : connectorProducerMap.entrySet()) {
+ ConnectorDescriptorId cdId = e.getKey();
+ Pair<IActivity, Integer> producer = e.getValue();
+ Pair<IActivity, Integer> consumer = connectorConsumerMap.get(cdId);
+ caMap.put(cdId, Pair.of(producer, consumer));
+ }
+ }
+
+ private <K, V> void addToValueSet(Map<K, Set<V>> map, K n1, V n2) {
+ Set<V> targets = map.get(n1);
+ if (targets == null) {
+ targets = new HashSet<V>();
+ map.put(n1, targets);
+ }
+ targets.add(n2);
+ }
+
+ private <T> void extend(List<T> list, int index) {
+ int n = list.size();
+ for (int i = n; i <= index; ++i) {
+ list.add(null);
+ }
+ }
+
+ private <K, V> void insertIntoIndexedMap(Map<K, List<V>> map, K key, int index, V value) {
+ List<V> vList = map.get(key);
+ if (vList == null) {
+ vList = new ArrayList<V>();
+ map.put(key, vList);
+ }
+ extend(vList, index);
+ vList.set(index, value);
+ }
+
+ public JobActivityGraph getActivityGraph() {
+ return jag;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/JobSpecificationActivityClusterGraphGeneratorFactory.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/JobSpecificationActivityClusterGraphGeneratorFactory.java
new file mode 100644
index 0000000..791f312
--- /dev/null
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/JobSpecificationActivityClusterGraphGeneratorFactory.java
@@ -0,0 +1,88 @@
+package edu.uci.ics.hyracks.api.client.impl;
+
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
+import edu.uci.ics.hyracks.api.constraints.Constraint;
+import edu.uci.ics.hyracks.api.constraints.IConstraintAcceptor;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.job.ActivityClusterGraph;
+import edu.uci.ics.hyracks.api.job.IActivityClusterGraphGenerator;
+import edu.uci.ics.hyracks.api.job.IActivityClusterGraphGeneratorFactory;
+import edu.uci.ics.hyracks.api.job.JobActivityGraph;
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class JobSpecificationActivityClusterGraphGeneratorFactory implements IActivityClusterGraphGeneratorFactory {
+ private static final long serialVersionUID = 1L;
+
+ private final JobSpecification spec;
+
+ public JobSpecificationActivityClusterGraphGeneratorFactory(JobSpecification jobSpec) {
+ this.spec = jobSpec;
+ }
+
+ @Override
+ public IActivityClusterGraphGenerator createActivityClusterGraphGenerator(String appName, JobId jobId,
+ final ICCApplicationContext ccAppCtx, EnumSet<JobFlag> jobFlags) throws HyracksException {
+ final JobActivityGraphBuilder builder = new JobActivityGraphBuilder(spec, jobFlags);
+ PlanUtils.visit(spec, new IConnectorDescriptorVisitor() {
+ @Override
+ public void visit(IConnectorDescriptor conn) throws HyracksException {
+ builder.addConnector(conn);
+ }
+ });
+ PlanUtils.visit(spec, new IOperatorDescriptorVisitor() {
+ @Override
+ public void visit(IOperatorDescriptor op) {
+ op.contributeActivities(builder);
+ }
+ });
+ builder.finish();
+ final JobActivityGraph jag = builder.getActivityGraph();
+ ActivityClusterGraphBuilder acgb = new ActivityClusterGraphBuilder();
+
+ final ActivityClusterGraph acg = acgb.inferActivityClusters(jobId, jag);
+ acg.setFrameSize(spec.getFrameSize());
+ acg.setMaxReattempts(spec.getMaxReattempts());
+ acg.setJobletEventListenerFactory(spec.getJobletEventListenerFactory());
+ acg.setGlobalJobDataFactory(spec.getGlobalJobDataFactory());
+ final Set<Constraint> constraints = new HashSet<Constraint>();
+ final IConstraintAcceptor acceptor = new IConstraintAcceptor() {
+ @Override
+ public void addConstraint(Constraint constraint) {
+ constraints.add(constraint);
+ }
+ };
+ PlanUtils.visit(spec, new IOperatorDescriptorVisitor() {
+ @Override
+ public void visit(IOperatorDescriptor op) {
+ op.contributeSchedulingConstraints(acceptor, ccAppCtx);
+ }
+ });
+ PlanUtils.visit(spec, new IConnectorDescriptorVisitor() {
+ @Override
+ public void visit(IConnectorDescriptor conn) {
+ conn.contributeSchedulingConstraints(acceptor, acg.getConnectorMap().get(conn.getConnectorId()),
+ ccAppCtx);
+ }
+ });
+ constraints.addAll(spec.getUserConstraints());
+ return new IActivityClusterGraphGenerator() {
+ @Override
+ public ActivityClusterGraph initialize() {
+ return acg;
+ }
+
+ @Override
+ public Set<Constraint> getConstraints() {
+ return constraints;
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/PlanUtils.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/PlanUtils.java
similarity index 97%
rename from hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/PlanUtils.java
rename to hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/PlanUtils.java
index 754094a..30f26c7 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/PlanUtils.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/impl/PlanUtils.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.control.cc.job;
+package edu.uci.ics.hyracks.api.client.impl;
import java.util.HashSet;
import java.util.Set;
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/Endpoint.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/Endpoint.java
deleted file mode 100644
index c0d23c1..0000000
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/Endpoint.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.api.comm;
-
-import java.io.Serializable;
-import java.util.UUID;
-
-public final class Endpoint implements Serializable {
- private static final long serialVersionUID = 1L;
-
- private final UUID id;
-
- private final NetworkAddress address;
-
- private final int receiverIndex;
-
- public Endpoint(NetworkAddress address, int receiverIndex) throws Exception {
- id = UUID.randomUUID();
- this.address = address;
- this.receiverIndex = receiverIndex;
- }
-
- public UUID getEndpointId() {
- return id;
- }
-
- public NetworkAddress getNetworkAddress() {
- return address;
- }
-
- public int getReceiverIndex() {
- return receiverIndex;
- }
-
- @Override
- public int hashCode() {
- return id.hashCode() + address.hashCode();
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof Endpoint)) {
- return false;
- }
- Endpoint oe = (Endpoint) o;
- return oe.id.equals(id) && oe.address.equals(address);
- }
-
- @Override
- public String toString() {
- return "[" + address + ":" + id + "]";
- }
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IConnectionDemultiplexer.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IConnectionDemultiplexer.java
deleted file mode 100644
index 0587722..0000000
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IConnectionDemultiplexer.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.api.comm;
-
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public interface IConnectionDemultiplexer {
- public int getSenderCount();
-
- public IConnectionEntry findNextReadyEntry(int lastReadSender);
-
- public void unreadyEntry(int index);
-
- public int closeEntry(int index) throws HyracksDataException;
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IConnectionEntry.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IConnectionEntry.java
deleted file mode 100644
index ec05d0b..0000000
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IConnectionEntry.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.api.comm;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.SelectionKey;
-import java.util.UUID;
-
-public interface IConnectionEntry {
- ByteBuffer getReadBuffer();
-
- SelectionKey getSelectionKey();
-
- void attach(Object attachment);
-
- Object getAttachment();
-
- void close() throws IOException;
-
- void write(ByteBuffer buffer);
-
- UUID getJobId();
-
- UUID getStageId();
-
- boolean aborted();
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IReceiver.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IReceiver.java
deleted file mode 100644
index 8ae10c4..0000000
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IReceiver.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.api.comm;
-
-import java.util.BitSet;
-
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-
-public interface IReceiver {
- public void setReceiverEventListener(IReceiverEventListener listener) throws HyracksException;
-
- public void init(int sender) throws HyracksException;
-
- public ISenderProxy getReadySender(BitSet senderMask, int senderIndexPreference);
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/ISender.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/ISender.java
deleted file mode 100644
index 88c8f27..0000000
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/ISender.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.api.comm;
-
-public interface ISender {
- public IFrameWriter createSenderWriter(int receiverIndex);
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/ICCContext.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/ICCContext.java
index 7466276..1e49fe2 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/ICCContext.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/ICCContext.java
@@ -18,9 +18,12 @@
import java.util.Set;
import edu.uci.ics.hyracks.api.client.ClusterControllerInfo;
+import edu.uci.ics.hyracks.api.topology.ClusterTopology;
public interface ICCContext {
public ClusterControllerInfo getClusterControllerInfo();
public void getIPAddressNodeMap(Map<String, Set<String>> map) throws Exception;
+
+ public ClusterTopology getClusterTopology();
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksCommonContext.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksCommonContext.java
index 62112f5..9cb8f10 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksCommonContext.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksCommonContext.java
@@ -19,9 +19,9 @@
import edu.uci.ics.hyracks.api.io.IIOManager;
public interface IHyracksCommonContext {
- public ByteBuffer allocateFrame();
-
public int getFrameSize();
public IIOManager getIOManager();
+
+ public ByteBuffer allocateFrame();
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksJobletContext.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksJobletContext.java
index e3c71ea..fad4300 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksJobletContext.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksJobletContext.java
@@ -20,10 +20,12 @@
import edu.uci.ics.hyracks.api.job.profiling.counters.ICounterContext;
import edu.uci.ics.hyracks.api.resources.IDeallocatableRegistry;
-public interface IHyracksJobletContext extends IHyracksCommonContext, IWorkspaceFileFactory, IDeallocatableRegistry {
+public interface IHyracksJobletContext extends IWorkspaceFileFactory, IDeallocatableRegistry {
public INCApplicationContext getApplicationContext();
public JobId getJobId();
public ICounterContext getCounterContext();
+
+ public Object getGlobalJobData();
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksRootContext.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksRootContext.java
index 2ad2e7f..a94c6de 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksRootContext.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksRootContext.java
@@ -14,5 +14,8 @@
*/
package edu.uci.ics.hyracks.api.context;
-public interface IHyracksRootContext extends IHyracksCommonContext {
+import edu.uci.ics.hyracks.api.io.IIOManager;
+
+public interface IHyracksRootContext {
+ public IIOManager getIOManager();
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksTaskContext.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksTaskContext.java
index 042d341..e964d66 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksTaskContext.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksTaskContext.java
@@ -27,4 +27,6 @@
public TaskAttemptId getTaskAttemptId();
public ICounterContext getCounterContext();
+
+ public void sendApplicationMessageToCC(byte[] message, String nodeId) throws Exception;
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IActivityGraphBuilder.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IActivityGraphBuilder.java
index 56870b2..bbda23c 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IActivityGraphBuilder.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IActivityGraphBuilder.java
@@ -15,7 +15,7 @@
package edu.uci.ics.hyracks.api.dataflow;
public interface IActivityGraphBuilder {
- public void addActivity(IActivity task);
+ public void addActivity(IOperatorDescriptor op, IActivity task);
public void addBlockingEdge(IActivity blocker, IActivity blocked);
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IConnectorDescriptor.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IConnectorDescriptor.java
index 0ac3658..3a6ee7a 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IConnectorDescriptor.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IConnectorDescriptor.java
@@ -28,7 +28,7 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobActivityGraph;
+import edu.uci.ics.hyracks.api.job.ActivityCluster;
/**
* Connector that connects operators in a Job.
@@ -91,10 +91,10 @@
*
* @param constraintAcceptor
* - Constraint Acceptor
- * @param plan
- * - Job Plan
+ * @param ac
+ * - Activity Cluster
*/
- public void contributeSchedulingConstraints(IConstraintAcceptor constraintAcceptor, JobActivityGraph plan,
+ public void contributeSchedulingConstraints(IConstraintAcceptor constraintAcceptor, ActivityCluster ac,
ICCApplicationContext appCtx);
/**
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorDescriptor.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorDescriptor.java
index c37a530..43bf141 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorDescriptor.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorDescriptor.java
@@ -22,7 +22,6 @@
import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
import edu.uci.ics.hyracks.api.constraints.IConstraintAcceptor;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobActivityGraph;
/**
* Descriptor for operators in Hyracks.
@@ -75,8 +74,7 @@
* @param plan
* - Job Plan
*/
- public void contributeSchedulingConstraints(IConstraintAcceptor constraintAcceptor, JobActivityGraph plan,
- ICCApplicationContext appCtx);
+ public void contributeSchedulingConstraints(IConstraintAcceptor constraintAcceptor, ICCApplicationContext appCtx);
/**
* Gets the display name.
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/state/ITaskState.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/state/IStateObject.java
similarity index 89%
rename from hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/state/ITaskState.java
rename to hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/state/IStateObject.java
index c24e488..f216773 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/state/ITaskState.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/state/IStateObject.java
@@ -18,13 +18,12 @@
import java.io.DataOutput;
import java.io.IOException;
-import edu.uci.ics.hyracks.api.dataflow.TaskId;
import edu.uci.ics.hyracks.api.job.JobId;
-public interface ITaskState {
+public interface IStateObject {
public JobId getJobId();
- public TaskId getTaskId();
+ public Object getId();
public long getMemoryOccupancy();
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IRecordDescriptorProvider.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IRecordDescriptorProvider.java
index 96d43a7..c480446 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IRecordDescriptorProvider.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IRecordDescriptorProvider.java
@@ -14,10 +14,10 @@
*/
package edu.uci.ics.hyracks.api.dataflow.value;
-import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.ActivityId;
public interface IRecordDescriptorProvider {
- public RecordDescriptor getInputRecordDescriptor(OperatorDescriptorId opId, int inputIndex);
+ public RecordDescriptor getInputRecordDescriptor(ActivityId aid, int inputIndex);
- public RecordDescriptor getOutputRecordDescriptor(OperatorDescriptorId opId, int outputIndex);
+ public RecordDescriptor getOutputRecordDescriptor(ActivityId aid, int outputIndex);
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITuplePairComparatorFactory.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITuplePairComparatorFactory.java
index 26cb525..297b56c 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITuplePairComparatorFactory.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITuplePairComparatorFactory.java
@@ -16,7 +16,8 @@
import java.io.Serializable;
-public interface ITuplePairComparatorFactory extends Serializable {
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
- public ITuplePairComparator createTuplePairComparator();
-}
+public interface ITuplePairComparatorFactory extends Serializable {
+ public ITuplePairComparator createTuplePairComparator(IHyracksTaskContext ctx);
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IReceiverEventListener.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/IFileHandle.java
similarity index 71%
copy from hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IReceiverEventListener.java
copy to hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/IFileHandle.java
index 98c083e..10fcef0 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IReceiverEventListener.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/IFileHandle.java
@@ -12,10 +12,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.api.comm;
+package edu.uci.ics.hyracks.api.io;
-public interface IReceiverEventListener {
- public void notifySenderAvailability(int sender);
-
- public void notifySenderCount(int senderCount);
+/**
+ * IFileHandle is an interface that exists only for Java compilers to perform static typing
+ * when handling file handle objects. Users must not implement this interface.
+ */
+public interface IFileHandle {
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/IIOManager.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/IIOManager.java
index 939cfc2..2e7feb5 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/IIOManager.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/IIOManager.java
@@ -33,16 +33,18 @@
public List<IODeviceHandle> getIODevices();
- public FileHandle open(FileReference fileRef, FileReadWriteMode rwMode, FileSyncMode syncMode)
+ public IFileHandle open(FileReference fileRef, FileReadWriteMode rwMode, FileSyncMode syncMode)
throws HyracksDataException;
- public int syncWrite(FileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException;
+ public int syncWrite(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException;
- public int syncRead(FileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException;
+ public int syncRead(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException;
- public IIOFuture asyncWrite(FileHandle fHandle, long offset, ByteBuffer data);
+ public IIOFuture asyncWrite(IFileHandle fHandle, long offset, ByteBuffer data);
- public IIOFuture asyncRead(FileHandle fHandle, long offset, ByteBuffer data);
+ public IIOFuture asyncRead(IFileHandle fHandle, long offset, ByteBuffer data);
- public void close(FileHandle fHandle) throws HyracksDataException;
+ public void close(IFileHandle fHandle) throws HyracksDataException;
+
+ public void sync(IFileHandle fileHandle, boolean metadata) throws HyracksDataException;
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/ActivityCluster.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/ActivityCluster.java
new file mode 100644
index 0000000..6698ff7
--- /dev/null
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/ActivityCluster.java
@@ -0,0 +1,232 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.job;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.api.dataflow.ActivityId;
+import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.IActivity;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicyAssignmentPolicy;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public final class ActivityCluster implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final ActivityClusterGraph acg;
+
+ private final ActivityClusterId id;
+
+ private final List<IActivity> roots;
+
+ private final Map<ActivityId, IActivity> activities;
+
+ private final Map<ConnectorDescriptorId, IConnectorDescriptor> connectors;
+
+ private final Map<ConnectorDescriptorId, RecordDescriptor> connectorRecordDescriptorMap;
+
+ private final Map<ActivityId, List<IConnectorDescriptor>> activityInputMap;
+
+ private final Map<ActivityId, List<IConnectorDescriptor>> activityOutputMap;
+
+ private final Map<ConnectorDescriptorId, Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>>> connectorActivityMap;
+
+ private final Map<ActivityId, Set<ActivityId>> blocked2blockerMap;
+
+ private final List<ActivityCluster> dependencies;
+
+ private IConnectorPolicyAssignmentPolicy cpap;
+
+ public ActivityCluster(ActivityClusterGraph acg, ActivityClusterId id) {
+ this.acg = acg;
+ this.id = id;
+ roots = new ArrayList<IActivity>();
+ activities = new HashMap<ActivityId, IActivity>();
+ connectors = new HashMap<ConnectorDescriptorId, IConnectorDescriptor>();
+ connectorRecordDescriptorMap = new HashMap<ConnectorDescriptorId, RecordDescriptor>();
+ activityInputMap = new HashMap<ActivityId, List<IConnectorDescriptor>>();
+ activityOutputMap = new HashMap<ActivityId, List<IConnectorDescriptor>>();
+ connectorActivityMap = new HashMap<ConnectorDescriptorId, Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>>>();
+ blocked2blockerMap = new HashMap<ActivityId, Set<ActivityId>>();
+ dependencies = new ArrayList<ActivityCluster>();
+ }
+
+ public ActivityClusterGraph getActivityClusterGraph() {
+ return acg;
+ }
+
+ public ActivityClusterId getId() {
+ return id;
+ }
+
+ public void addRoot(IActivity activity) {
+ roots.add(activity);
+ }
+
+ public void addActivity(IActivity activity) {
+ activities.put(activity.getActivityId(), activity);
+ }
+
+ public void addConnector(IConnectorDescriptor connector) {
+ connectors.put(connector.getConnectorId(), connector);
+ }
+
+ public void connect(IConnectorDescriptor connector, IActivity producerActivity, int producerPort,
+ IActivity consumerActivity, int consumerPort, RecordDescriptor recordDescriptor) {
+ if (!activities.containsKey(producerActivity.getActivityId())
+ || !activities.containsKey(consumerActivity.getActivityId())) {
+ throw new IllegalStateException("Connected Activities belong to different Activity Clusters: "
+ + producerActivity.getActivityId() + " and " + consumerActivity.getActivityId());
+ }
+ insertIntoIndexedMap(activityInputMap, consumerActivity.getActivityId(), consumerPort, connector);
+ insertIntoIndexedMap(activityOutputMap, producerActivity.getActivityId(), producerPort, connector);
+ connectorActivityMap.put(
+ connector.getConnectorId(),
+ Pair.<Pair<IActivity, Integer>, Pair<IActivity, Integer>> of(
+ Pair.<IActivity, Integer> of(producerActivity, producerPort),
+ Pair.<IActivity, Integer> of(consumerActivity, consumerPort)));
+ connectorRecordDescriptorMap.put(connector.getConnectorId(), recordDescriptor);
+ }
+
+ public List<IActivity> getRoots() {
+ return roots;
+ }
+
+ public Map<ActivityId, IActivity> getActivityMap() {
+ return activities;
+ }
+
+ public Map<ConnectorDescriptorId, IConnectorDescriptor> getConnectorMap() {
+ return connectors;
+ }
+
+ public Map<ConnectorDescriptorId, RecordDescriptor> getConnectorRecordDescriptorMap() {
+ return connectorRecordDescriptorMap;
+ }
+
+ public Map<ConnectorDescriptorId, Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>>> getConnectorActivityMap() {
+ return connectorActivityMap;
+ }
+
+ public Map<ActivityId, List<IConnectorDescriptor>> getActivityInputMap() {
+ return activityInputMap;
+ }
+
+ public Map<ActivityId, List<IConnectorDescriptor>> getActivityOutputMap() {
+ return activityOutputMap;
+ }
+
+ public ActivityId getConsumerActivity(ConnectorDescriptorId cdId) {
+ Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>> connEdge = connectorActivityMap.get(cdId);
+ return connEdge.getRight().getLeft().getActivityId();
+ }
+
+ public ActivityId getProducerActivity(ConnectorDescriptorId cdId) {
+ Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>> connEdge = connectorActivityMap.get(cdId);
+ return connEdge.getLeft().getLeft().getActivityId();
+ }
+
+ public Map<ActivityId, Set<ActivityId>> getBlocked2BlockerMap() {
+ return blocked2blockerMap;
+ }
+
+ public List<ActivityCluster> getDependencies() {
+ return dependencies;
+ }
+
+ public IConnectorPolicyAssignmentPolicy getConnectorPolicyAssignmentPolicy() {
+ return cpap;
+ }
+
+ public void setConnectorPolicyAssignmentPolicy(IConnectorPolicyAssignmentPolicy cpap) {
+ this.cpap = cpap;
+ }
+
+ private <T> void extend(List<T> list, int index) {
+ int n = list.size();
+ for (int i = n; i <= index; ++i) {
+ list.add(null);
+ }
+ }
+
+ private <K, V> void insertIntoIndexedMap(Map<K, List<V>> map, K key, int index, V value) {
+ List<V> vList = map.get(key);
+ if (vList == null) {
+ vList = new ArrayList<V>();
+ map.put(key, vList);
+ }
+ extend(vList, index);
+ vList.set(index, value);
+ }
+
+ public JSONObject toJSON() throws JSONException {
+ JSONObject jac = new JSONObject();
+
+ JSONArray jans = new JSONArray();
+ for (IActivity an : activities.values()) {
+ JSONObject jan = new JSONObject();
+ jan.put("id", an.getActivityId().toString());
+ jan.put("java-class", an.getClass().getName());
+
+ List<IConnectorDescriptor> inputs = activityInputMap.get(an.getActivityId());
+ if (inputs != null) {
+ JSONArray jInputs = new JSONArray();
+ for (int i = 0; i < inputs.size(); ++i) {
+ JSONObject jInput = new JSONObject();
+ jInput.put("input-port", i);
+ jInput.put("connector-id", inputs.get(i).getConnectorId().toString());
+ jInputs.put(jInput);
+ }
+ jan.put("inputs", jInputs);
+ }
+
+ List<IConnectorDescriptor> outputs = activityOutputMap.get(an.getActivityId());
+ if (outputs != null) {
+ JSONArray jOutputs = new JSONArray();
+ for (int i = 0; i < outputs.size(); ++i) {
+ JSONObject jOutput = new JSONObject();
+ jOutput.put("output-port", i);
+ jOutput.put("connector-id", outputs.get(i).getConnectorId().toString());
+ jOutputs.put(jOutput);
+ }
+ jan.put("outputs", jOutputs);
+ }
+
+ Set<ActivityId> blockers = getBlocked2BlockerMap().get(an.getActivityId());
+ if (blockers != null) {
+ JSONArray jDeps = new JSONArray();
+ for (ActivityId blocker : blockers) {
+ jDeps.put(blocker.toString());
+ }
+ jan.put("depends-on", jDeps);
+ }
+ jans.put(jan);
+ }
+ jac.put("activities", jans);
+
+ return jac;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/ActivityClusterGraph.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/ActivityClusterGraph.java
new file mode 100644
index 0000000..dd68747
--- /dev/null
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/ActivityClusterGraph.java
@@ -0,0 +1,128 @@
+/*
+2 * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.job;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.api.dataflow.ActivityId;
+import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
+
+public class ActivityClusterGraph implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private int version;
+
+ private final Map<ActivityClusterId, ActivityCluster> activityClusterMap;
+
+ private final Map<ActivityId, ActivityCluster> activityMap;
+
+ private final Map<ConnectorDescriptorId, ActivityCluster> connectorMap;
+
+ private int frameSize;
+
+ private int maxReattempts;
+
+ private IJobletEventListenerFactory jobletEventListenerFactory;
+
+ private IGlobalJobDataFactory globalJobDataFactory;
+
+ public ActivityClusterGraph() {
+ version = 0;
+ activityClusterMap = new HashMap<ActivityClusterId, ActivityCluster>();
+ activityMap = new HashMap<ActivityId, ActivityCluster>();
+ connectorMap = new HashMap<ConnectorDescriptorId, ActivityCluster>();
+ frameSize = 32768;
+ }
+
+ public Map<ActivityId, ActivityCluster> getActivityMap() {
+ return activityMap;
+ }
+
+ public Map<ConnectorDescriptorId, ActivityCluster> getConnectorMap() {
+ return connectorMap;
+ }
+
+ public Map<ActivityClusterId, ActivityCluster> getActivityClusterMap() {
+ return activityClusterMap;
+ }
+
+ public void addActivityClusters(Collection<ActivityCluster> newActivityClusters) {
+ for (ActivityCluster ac : newActivityClusters) {
+ activityClusterMap.put(ac.getId(), ac);
+ for (ActivityId aid : ac.getActivityMap().keySet()) {
+ activityMap.put(aid, ac);
+ }
+ for (ConnectorDescriptorId cid : ac.getConnectorMap().keySet()) {
+ connectorMap.put(cid, ac);
+ }
+ }
+ ++version;
+ }
+
+ public int getVersion() {
+ return version;
+ }
+
+ public void setFrameSize(int frameSize) {
+ this.frameSize = frameSize;
+ }
+
+ public int getFrameSize() {
+ return frameSize;
+ }
+
+ public void setMaxReattempts(int maxReattempts) {
+ this.maxReattempts = maxReattempts;
+ }
+
+ public int getMaxReattempts() {
+ return maxReattempts;
+ }
+
+ public IJobletEventListenerFactory getJobletEventListenerFactory() {
+ return jobletEventListenerFactory;
+ }
+
+ public void setJobletEventListenerFactory(IJobletEventListenerFactory jobletEventListenerFactory) {
+ this.jobletEventListenerFactory = jobletEventListenerFactory;
+ }
+
+ public IGlobalJobDataFactory getGlobalJobDataFactory() {
+ return globalJobDataFactory;
+ }
+
+ public void setGlobalJobDataFactory(IGlobalJobDataFactory globalJobDataFactory) {
+ this.globalJobDataFactory = globalJobDataFactory;
+ }
+
+ public JSONObject toJSON() throws JSONException {
+ JSONObject acgj = new JSONObject();
+
+ JSONArray acl = new JSONArray();
+ for (ActivityCluster ac : activityClusterMap.values()) {
+ acl.put(ac.toJSON());
+ }
+ acgj.put("version", version);
+ acgj.put("activity-clusters", acl);
+ return acgj;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/ActivityClusterId.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/ActivityClusterId.java
similarity index 66%
rename from hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/ActivityClusterId.java
rename to hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/ActivityClusterId.java
index 46628a4..cba63959 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/ActivityClusterId.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/ActivityClusterId.java
@@ -12,19 +12,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.control.cc.job;
+package edu.uci.ics.hyracks.api.job;
import java.io.Serializable;
public final class ActivityClusterId implements Serializable {
private static final long serialVersionUID = 1L;
+ private final JobId jobId;
+
private final int id;
- public ActivityClusterId(int id) {
+ public ActivityClusterId(JobId jobId, int id) {
+ this.jobId = jobId;
this.id = id;
}
+ public JobId getJobId() {
+ return jobId;
+ }
+
public int getId() {
return id;
}
@@ -34,25 +41,37 @@
final int prime = 31;
int result = 1;
result = prime * result + id;
+ result = prime * result + ((jobId == null) ? 0 : jobId.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
- if (this == obj)
+ if (this == obj) {
return true;
- if (obj == null)
+ }
+ if (obj == null) {
return false;
- if (getClass() != obj.getClass())
+ }
+ if (getClass() != obj.getClass()) {
return false;
+ }
ActivityClusterId other = (ActivityClusterId) obj;
- if (id != other.id)
+ if (id != other.id) {
return false;
+ }
+ if (jobId == null) {
+ if (other.jobId != null) {
+ return false;
+ }
+ } else if (!jobId.equals(other.jobId)) {
+ return false;
+ }
return true;
}
@Override
public String toString() {
- return "AC:" + id;
+ return "ACID:" + jobId + ":" + id;
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/ISenderProxy.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IActivityClusterGraphGenerator.java
similarity index 71%
rename from hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/ISenderProxy.java
rename to hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IActivityClusterGraphGenerator.java
index 6b40bae..b837066 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/ISenderProxy.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IActivityClusterGraphGenerator.java
@@ -12,12 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.api.comm;
+package edu.uci.ics.hyracks.api.job;
-import java.nio.ByteBuffer;
+import java.util.Set;
-public interface ISenderProxy {
- public ByteBuffer getReadBuffer();
+import edu.uci.ics.hyracks.api.constraints.Constraint;
- public int getSenderIndex();
+public interface IActivityClusterGraphGenerator {
+ public Set<Constraint> getConstraints();
+
+ public ActivityClusterGraph initialize();
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IJobSpecificationFactory.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IActivityClusterGraphGeneratorFactory.java
similarity index 70%
rename from hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IJobSpecificationFactory.java
rename to hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IActivityClusterGraphGeneratorFactory.java
index 722aaae..d801dd1 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IJobSpecificationFactory.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IActivityClusterGraphGeneratorFactory.java
@@ -14,11 +14,13 @@
*/
package edu.uci.ics.hyracks.api.job;
+import java.io.Serializable;
+import java.util.EnumSet;
+
import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
-import edu.uci.ics.hyracks.api.application.ICCBootstrap;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-public interface IJobSpecificationFactory {
- public JobSpecification createJobSpecification(byte[] bytes, ICCBootstrap bootstrap, ICCApplicationContext appCtx)
- throws HyracksException;
+public interface IActivityClusterGraphGeneratorFactory extends Serializable {
+ public IActivityClusterGraphGenerator createActivityClusterGraphGenerator(String appName, JobId jobId,
+ ICCApplicationContext ccAppCtx, EnumSet<JobFlag> jobFlags) throws HyracksException;
}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/IConnectorDescriptorVisitor.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IConnectorDescriptorRegistry.java
similarity index 74%
copy from hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/IConnectorDescriptorVisitor.java
copy to hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IConnectorDescriptorRegistry.java
index 1c8446f..65b27e3 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/IConnectorDescriptorVisitor.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IConnectorDescriptorRegistry.java
@@ -12,11 +12,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.control.cc.job;
+package edu.uci.ics.hyracks.api.job;
+import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-public interface IConnectorDescriptorVisitor {
- public void visit(IConnectorDescriptor conn) throws HyracksException;
+public interface IConnectorDescriptorRegistry {
+ public ConnectorDescriptorId createConnectorDescriptor(IConnectorDescriptor conn);
}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IGlobalJobDataFactory.java
similarity index 74%
copy from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
copy to hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IGlobalJobDataFactory.java
index 871fe43..bee495a 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IGlobalJobDataFactory.java
@@ -12,10 +12,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+package edu.uci.ics.hyracks.api.job;
import java.io.Serializable;
-public interface IPrinterFactory extends Serializable {
- public IPrinter createPrinter();
-}
+import edu.uci.ics.hyracks.api.context.IHyracksJobletContext;
+
+public interface IGlobalJobDataFactory extends Serializable {
+ public Object createGlobalJobData(IHyracksJobletContext ctx);
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IJobLifecycleListener.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IJobLifecycleListener.java
index c3a0f7e..2f0f025 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IJobLifecycleListener.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IJobLifecycleListener.java
@@ -17,7 +17,7 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
public interface IJobLifecycleListener {
- public void notifyJobCreation(JobId jobId, JobSpecification jobSpec) throws HyracksException;
+ public void notifyJobCreation(JobId jobId, IActivityClusterGraphGeneratorFactory acggf) throws HyracksException;
public void notifyJobStart(JobId jobId) throws HyracksException;
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/IOperatorDescriptorVisitor.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IOperatorDescriptorRegistry.java
similarity index 74%
copy from hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/IOperatorDescriptorVisitor.java
copy to hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IOperatorDescriptorRegistry.java
index 840faef..4ea84d6 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/IOperatorDescriptorVisitor.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IOperatorDescriptorRegistry.java
@@ -12,11 +12,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.control.cc.job;
+package edu.uci.ics.hyracks.api.job;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
-public interface IOperatorDescriptorVisitor {
- public void visit(IOperatorDescriptor op) throws HyracksException;
+public interface IOperatorDescriptorRegistry {
+ public OperatorDescriptorId createOperatorDescriptorId(IOperatorDescriptor op);
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IOperatorEnvironment.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IOperatorEnvironment.java
index ecc44c1..eaa1ec2 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IOperatorEnvironment.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IOperatorEnvironment.java
@@ -14,11 +14,10 @@
*/
package edu.uci.ics.hyracks.api.job;
-import edu.uci.ics.hyracks.api.dataflow.TaskId;
-import edu.uci.ics.hyracks.api.dataflow.state.ITaskState;
+import edu.uci.ics.hyracks.api.dataflow.state.IStateObject;
public interface IOperatorEnvironment {
- public void setTaskState(ITaskState taskState);
+ public void setStateObject(IStateObject taskState);
- public ITaskState getTaskState(TaskId taskId);
+ public IStateObject getStateObject(Object id);
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobActivityGraph.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobActivityGraph.java
index 2b6d361..5b62b5c 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobActivityGraph.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobActivityGraph.java
@@ -15,245 +15,91 @@
package edu.uci.ics.hyracks.api.job;
import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.tuple.Pair;
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
import edu.uci.ics.hyracks.api.dataflow.ActivityId;
import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
import edu.uci.ics.hyracks.api.dataflow.IActivity;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
public class JobActivityGraph implements Serializable {
private static final long serialVersionUID = 1L;
- private final String appName;
+ private final Map<ActivityId, IActivity> activityMap;
- private final JobSpecification jobSpec;
+ private final Map<ConnectorDescriptorId, IConnectorDescriptor> connectorMap;
- private final EnumSet<JobFlag> jobFlags;
+ private final Map<ConnectorDescriptorId, RecordDescriptor> connectorRecordDescriptorMap;
- private final Map<ActivityId, IActivity> activityNodes;
+ private final Map<ActivityId, List<IConnectorDescriptor>> activityInputMap;
- private final Map<ActivityId, Set<ActivityId>> blocker2blockedMap;
+ private final Map<ActivityId, List<IConnectorDescriptor>> activityOutputMap;
+
+ private final Map<ConnectorDescriptorId, Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>>> connectorActivityMap;
private final Map<ActivityId, Set<ActivityId>> blocked2blockerMap;
- private final Map<OperatorDescriptorId, Set<ActivityId>> operatorActivityMap;
-
- private final Map<ActivityId, List<Integer>> activityInputMap;
-
- private final Map<ActivityId, List<Integer>> activityOutputMap;
-
- private final Map<OperatorDescriptorId, List<ActivityId>> operatorInputMap;
-
- private final Map<OperatorDescriptorId, List<ActivityId>> operatorOutputMap;
-
- public JobActivityGraph(String appName, JobSpecification jobSpec, EnumSet<JobFlag> jobFlags) {
- this.appName = appName;
- this.jobSpec = jobSpec;
- this.jobFlags = jobFlags;
- activityNodes = new HashMap<ActivityId, IActivity>();
- blocker2blockedMap = new HashMap<ActivityId, Set<ActivityId>>();
+ public JobActivityGraph() {
+ activityMap = new HashMap<ActivityId, IActivity>();
+ connectorMap = new HashMap<ConnectorDescriptorId, IConnectorDescriptor>();
+ connectorRecordDescriptorMap = new HashMap<ConnectorDescriptorId, RecordDescriptor>();
+ activityInputMap = new HashMap<ActivityId, List<IConnectorDescriptor>>();
+ activityOutputMap = new HashMap<ActivityId, List<IConnectorDescriptor>>();
+ connectorActivityMap = new HashMap<ConnectorDescriptorId, Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>>>();
blocked2blockerMap = new HashMap<ActivityId, Set<ActivityId>>();
- operatorActivityMap = new HashMap<OperatorDescriptorId, Set<ActivityId>>();
- activityInputMap = new HashMap<ActivityId, List<Integer>>();
- activityOutputMap = new HashMap<ActivityId, List<Integer>>();
- operatorInputMap = new HashMap<OperatorDescriptorId, List<ActivityId>>();
- operatorOutputMap = new HashMap<OperatorDescriptorId, List<ActivityId>>();
}
- public String getApplicationName() {
- return appName;
+ public Map<ActivityId, IActivity> getActivityMap() {
+ return activityMap;
}
- public JobSpecification getJobSpecification() {
- return jobSpec;
+ public Map<ConnectorDescriptorId, IConnectorDescriptor> getConnectorMap() {
+ return connectorMap;
}
- public EnumSet<JobFlag> getJobFlags() {
- return jobFlags;
- }
-
- public Map<ActivityId, IActivity> getActivityNodeMap() {
- return activityNodes;
- }
-
- public Map<ActivityId, Set<ActivityId>> getBlocker2BlockedMap() {
- return blocker2blockedMap;
+ public Map<ConnectorDescriptorId, RecordDescriptor> getConnectorRecordDescriptorMap() {
+ return connectorRecordDescriptorMap;
}
public Map<ActivityId, Set<ActivityId>> getBlocked2BlockerMap() {
return blocked2blockerMap;
}
- public Map<OperatorDescriptorId, Set<ActivityId>> getOperatorActivityMap() {
- return operatorActivityMap;
- }
-
- public Map<ActivityId, List<Integer>> getActivityInputMap() {
+ public Map<ActivityId, List<IConnectorDescriptor>> getActivityInputMap() {
return activityInputMap;
}
- public Map<ActivityId, List<Integer>> getActivityOutputMap() {
+ public Map<ActivityId, List<IConnectorDescriptor>> getActivityOutputMap() {
return activityOutputMap;
}
- public Map<OperatorDescriptorId, List<ActivityId>> getOperatorInputMap() {
- return operatorInputMap;
- }
-
- public Map<OperatorDescriptorId, List<ActivityId>> getOperatorOutputMap() {
- return operatorOutputMap;
- }
-
- public List<IConnectorDescriptor> getActivityInputConnectorDescriptors(ActivityId hanId) {
- List<Integer> inputIndexes = activityInputMap.get(hanId);
- if (inputIndexes == null) {
- return null;
- }
- OperatorDescriptorId ownerId = hanId.getOperatorDescriptorId();
- List<IConnectorDescriptor> inputs = new ArrayList<IConnectorDescriptor>();
- for (Integer i : inputIndexes) {
- inputs.add(jobSpec.getInputConnectorDescriptor(ownerId, i));
- }
- return inputs;
- }
-
- public List<IConnectorDescriptor> getActivityOutputConnectorDescriptors(ActivityId hanId) {
- List<Integer> outputIndexes = activityOutputMap.get(hanId);
- if (outputIndexes == null) {
- return null;
- }
- OperatorDescriptorId ownerId = hanId.getOperatorDescriptorId();
- List<IConnectorDescriptor> outputs = new ArrayList<IConnectorDescriptor>();
- for (Integer i : outputIndexes) {
- outputs.add(jobSpec.getOutputConnectorDescriptor(ownerId, i));
- }
- return outputs;
+ public Map<ConnectorDescriptorId, Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>>> getConnectorActivityMap() {
+ return connectorActivityMap;
}
public ActivityId getConsumerActivity(ConnectorDescriptorId cdId) {
- Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connEdge = jobSpec
- .getConnectorOperatorMap().get(cdId);
-
- OperatorDescriptorId consumerOpId = connEdge.getRight().getLeft().getOperatorId();
- int consumerInputIdx = connEdge.getRight().getRight();
-
- for (ActivityId anId : operatorActivityMap.get(consumerOpId)) {
- List<Integer> anInputs = activityInputMap.get(anId);
- if (anInputs != null) {
- for (Integer idx : anInputs) {
- if (idx.intValue() == consumerInputIdx) {
- return anId;
- }
- }
- }
- }
- return null;
+ Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>> connEdge = connectorActivityMap.get(cdId);
+ return connEdge.getRight().getLeft().getActivityId();
}
public ActivityId getProducerActivity(ConnectorDescriptorId cdId) {
- Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connEdge = jobSpec
- .getConnectorOperatorMap().get(cdId);
-
- OperatorDescriptorId producerOpId = connEdge.getLeft().getLeft().getOperatorId();
- int producerInputIdx = connEdge.getLeft().getRight();
-
- for (ActivityId anId : operatorActivityMap.get(producerOpId)) {
- List<Integer> anOutputs = activityOutputMap.get(anId);
- if (anOutputs != null) {
- for (Integer idx : anOutputs) {
- if (idx.intValue() == producerInputIdx) {
- return anId;
- }
- }
- }
- }
- return null;
- }
-
- public RecordDescriptor getActivityInputRecordDescriptor(ActivityId hanId, int inputIndex) {
- int opInputIndex = getActivityInputMap().get(hanId).get(inputIndex);
- return jobSpec.getOperatorInputRecordDescriptor(hanId.getOperatorDescriptorId(), opInputIndex);
- }
-
- public RecordDescriptor getActivityOutputRecordDescriptor(ActivityId hanId, int outputIndex) {
- int opOutputIndex = getActivityOutputMap().get(hanId).get(outputIndex);
- return jobSpec.getOperatorOutputRecordDescriptor(hanId.getOperatorDescriptorId(), opOutputIndex);
+ Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>> connEdge = connectorActivityMap.get(cdId);
+ return connEdge.getLeft().getLeft().getActivityId();
}
@Override
public String toString() {
StringBuilder buffer = new StringBuilder();
- buffer.append("ActivityNodes: " + activityNodes);
+ buffer.append("ActivityNodes: " + activityMap);
buffer.append('\n');
- buffer.append("Blocker->Blocked: " + blocker2blockedMap);
- buffer.append('\n');
- buffer.append("Blocked->Blocker: " + blocked2blockerMap);
+ buffer.append("Blocker->Blocked: " + blocked2blockerMap);
buffer.append('\n');
return buffer.toString();
}
-
- public JSONObject toJSON() throws JSONException {
- JSONObject jplan = new JSONObject();
-
- jplan.put("flags", jobFlags.toString());
-
- JSONArray jans = new JSONArray();
- for (IActivity an : activityNodes.values()) {
- JSONObject jan = new JSONObject();
- jan.put("id", an.getActivityId().toString());
- jan.put("java-class", an.getClass().getName());
- jan.put("operator-id", an.getActivityId().getOperatorDescriptorId().toString());
-
- List<IConnectorDescriptor> inputs = getActivityInputConnectorDescriptors(an.getActivityId());
- if (inputs != null) {
- JSONArray jInputs = new JSONArray();
- for (int i = 0; i < inputs.size(); ++i) {
- JSONObject jInput = new JSONObject();
- jInput.put("input-port", i);
- jInput.put("connector-id", inputs.get(i).getConnectorId().toString());
- jInputs.put(jInput);
- }
- jan.put("inputs", jInputs);
- }
-
- List<IConnectorDescriptor> outputs = getActivityOutputConnectorDescriptors(an.getActivityId());
- if (outputs != null) {
- JSONArray jOutputs = new JSONArray();
- for (int i = 0; i < outputs.size(); ++i) {
- JSONObject jOutput = new JSONObject();
- jOutput.put("output-port", i);
- jOutput.put("connector-id", outputs.get(i).getConnectorId().toString());
- jOutputs.put(jOutput);
- }
- jan.put("outputs", jOutputs);
- }
-
- Set<ActivityId> blockers = getBlocked2BlockerMap().get(an.getActivityId());
- if (blockers != null) {
- JSONArray jDeps = new JSONArray();
- for (ActivityId blocker : blockers) {
- jDeps.put(blocker.toString());
- }
- jan.put("depends-on", jDeps);
- }
- jans.put(jan);
- }
- jplan.put("activities", jans);
-
- return jplan;
- }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java
index 90c799d..cf2eec2 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java
@@ -35,7 +35,7 @@
import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicyAssignmentPolicy;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-public class JobSpecification implements Serializable {
+public class JobSpecification implements Serializable, IOperatorDescriptorRegistry, IConnectorDescriptorRegistry {
private static final long serialVersionUID = 1L;
private final List<OperatorDescriptorId> roots;
@@ -56,10 +56,14 @@
private IConnectorPolicyAssignmentPolicy connectorPolicyAssignmentPolicy;
+ private int frameSize;
+
private int maxReattempts;
private IJobletEventListenerFactory jobletEventListenerFactory;
+ private IGlobalJobDataFactory globalJobDataFactory;
+
private transient int operatorIdCounter;
private transient int connectorIdCounter;
@@ -75,15 +79,22 @@
userConstraints = new HashSet<Constraint>();
operatorIdCounter = 0;
connectorIdCounter = 0;
+ frameSize = 32768;
maxReattempts = 2;
}
- public OperatorDescriptorId createOperatorDescriptorId() {
- return new OperatorDescriptorId(operatorIdCounter++);
+ @Override
+ public OperatorDescriptorId createOperatorDescriptorId(IOperatorDescriptor op) {
+ OperatorDescriptorId odId = new OperatorDescriptorId(operatorIdCounter++);
+ opMap.put(odId, op);
+ return odId;
}
- public ConnectorDescriptorId createConnectorDescriptor() {
- return new ConnectorDescriptorId(connectorIdCounter++);
+ @Override
+ public ConnectorDescriptorId createConnectorDescriptor(IConnectorDescriptor conn) {
+ ConnectorDescriptorId cdId = new ConnectorDescriptorId(connectorIdCounter++);
+ connMap.put(cdId, conn);
+ return cdId;
}
public void addRoot(IOperatorDescriptor op) {
@@ -202,6 +213,14 @@
this.connectorPolicyAssignmentPolicy = connectorPolicyAssignmentPolicy;
}
+ public void setFrameSize(int frameSize) {
+ this.frameSize = frameSize;
+ }
+
+ public int getFrameSize() {
+ return frameSize;
+ }
+
public void setMaxReattempts(int maxReattempts) {
this.maxReattempts = maxReattempts;
}
@@ -226,6 +245,14 @@
this.jobletEventListenerFactory = jobletEventListenerFactory;
}
+ public IGlobalJobDataFactory getGlobalJobDataFactory() {
+ return globalJobDataFactory;
+ }
+
+ public void setGlobalJobDataFactory(IGlobalJobDataFactory globalJobDataFactory) {
+ this.globalJobDataFactory = globalJobDataFactory;
+ }
+
private <K, V> void insertIntoIndexedMap(Map<K, List<V>> map, K key, int index, V value) {
List<V> vList = map.get(key);
if (vList == null) {
@@ -259,6 +286,8 @@
}
}
+ buffer.append("\n").append("Constraints:\n").append(userConstraints);
+
return buffer.toString();
}
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/things/IThingDescriptor.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/messages/IMessage.java
similarity index 82%
rename from hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/things/IThingDescriptor.java
rename to hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/messages/IMessage.java
index 1269e35..96ad7fe 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/things/IThingDescriptor.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/messages/IMessage.java
@@ -12,10 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.api.things;
+package edu.uci.ics.hyracks.api.messages;
import java.io.Serializable;
-public interface IThingDescriptor extends Serializable {
- public ThingDescriptorId getThingId();
-}
\ No newline at end of file
+/**
+ * @author rico
+ *
+ */
+public interface IMessage extends Serializable {
+
+}
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/things/IThingDescriptor.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/messages/IMessageBroker.java
similarity index 78%
copy from hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/things/IThingDescriptor.java
copy to hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/messages/IMessageBroker.java
index 1269e35..35ae813 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/things/IThingDescriptor.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/messages/IMessageBroker.java
@@ -12,10 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.api.things;
+package edu.uci.ics.hyracks.api.messages;
-import java.io.Serializable;
+/**
+ * @author rico
+ *
+ */
+public interface IMessageBroker {
-public interface IThingDescriptor extends Serializable {
- public ThingDescriptorId getThingId();
-}
\ No newline at end of file
+ public void receivedMessage(IMessage message, String nodeId);
+
+}
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/naming/MultipartName.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/naming/MultipartName.java
deleted file mode 100644
index 7310606..0000000
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/naming/MultipartName.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.api.naming;
-
-import java.util.Arrays;
-
-public final class MultipartName {
- private Object[] parts;
-
- public MultipartName(Object... parts) {
- this.parts = parts;
- }
-
- public Object[] getParts() {
- return parts;
- }
-
- @Override
- public int hashCode() {
- return Arrays.deepHashCode(parts);
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof MultipartName)) {
- return false;
- }
- return Arrays.deepEquals(parts, ((MultipartName) o).parts);
- }
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/partitions/IPartition.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/partitions/IPartition.java
index 02eb891..c3dfad0 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/partitions/IPartition.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/partitions/IPartition.java
@@ -15,9 +15,12 @@
package edu.uci.ics.hyracks.api.partitions;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.resources.IDeallocatable;
public interface IPartition extends IDeallocatable {
+ public IHyracksTaskContext getTaskContext();
+
public void writeTo(IFrameWriter writer);
public boolean isReusable();
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/things/ISliver.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/things/ISliver.java
deleted file mode 100644
index 135a77c..0000000
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/things/ISliver.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.api.things;
-
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public interface ISliver {
- public void open() throws HyracksDataException;
-
- public void nextFrame(ByteBuffer buffer) throws HyracksDataException;
-
- public void commit() throws HyracksDataException;
-
- public void abort();
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/things/ThingDescriptorId.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/things/ThingDescriptorId.java
deleted file mode 100644
index 5d42ce9..0000000
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/things/ThingDescriptorId.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.api.things;
-
-import java.io.Serializable;
-
-public final class ThingDescriptorId implements Serializable {
- private static final long serialVersionUID = 1L;
-
- private final long id;
-
- public ThingDescriptorId(long id) {
- this.id = id;
- }
-
- public long getId() {
- return id;
- }
-
- @Override
- public String toString() {
- return "ThID: " + id;
- }
-
- public int hashCode() {
- return (int) (id & 0xffffffff);
- }
-
- public boolean equals(Object o) {
- if (!(o instanceof ThingDescriptorId)) {
- return false;
- }
- return id == ((ThingDescriptorId) o).id;
- }
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/topology/ClusterTopology.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/topology/ClusterTopology.java
new file mode 100644
index 0000000..6f566d1
--- /dev/null
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/topology/ClusterTopology.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.topology;
+
+import java.io.Serializable;
+import java.util.List;
+
+public class ClusterTopology implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final NetworkSwitch clusterSwitch;
+
+ public ClusterTopology(NetworkSwitch clusterSwitch) {
+ this.clusterSwitch = clusterSwitch;
+ }
+
+ public NetworkSwitch getClusterSwitch() {
+ return clusterSwitch;
+ }
+
+ public boolean lookupNetworkTerminal(String terminalName, List<Integer> path) {
+ return clusterSwitch.lookupNetworkTerminal(terminalName, path);
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/topology/NetworkEndpoint.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/topology/NetworkEndpoint.java
new file mode 100644
index 0000000..0aaee2a
--- /dev/null
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/topology/NetworkEndpoint.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.topology;
+
+import java.io.Serializable;
+import java.util.Map;
+
+public abstract class NetworkEndpoint implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ public enum EndpointType {
+ NETWORK_SWITCH,
+ NETWORK_TERMINAL,
+ }
+
+ protected final String name;
+
+ protected final Map<String, String> properties;
+
+ public NetworkEndpoint(String name, Map<String, String> properties) {
+ this.name = name;
+ this.properties = properties;
+ }
+
+ public abstract EndpointType getType();
+
+ public final String getName() {
+ return name;
+ }
+
+ public final Map<String, String> getProperties() {
+ return properties;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/topology/NetworkSwitch.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/topology/NetworkSwitch.java
new file mode 100644
index 0000000..3c89044
--- /dev/null
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/topology/NetworkSwitch.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.topology;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class NetworkSwitch extends NetworkEndpoint {
+ private static final long serialVersionUID = 1L;
+
+ private final Port[] ports;
+
+ private final Map<String, Integer> terminalNamePortIndexMap;
+
+ public NetworkSwitch(String name, Map<String, String> properties, Port[] ports) {
+ super(name, properties);
+ this.ports = ports;
+ terminalNamePortIndexMap = new HashMap<String, Integer>();
+ for (int i = 0; i < ports.length; ++i) {
+ Port port = ports[i];
+ NetworkEndpoint endpoint = port.getEndpoint();
+ Integer portIndex = Integer.valueOf(i);
+ switch (endpoint.getType()) {
+ case NETWORK_SWITCH: {
+ NetworkSwitch s = (NetworkSwitch) endpoint;
+ for (String t : s.terminalNamePortIndexMap.keySet()) {
+ terminalNamePortIndexMap.put(t, portIndex);
+ }
+ break;
+ }
+
+ case NETWORK_TERMINAL: {
+ NetworkTerminal t = (NetworkTerminal) endpoint;
+ terminalNamePortIndexMap.put(t.getName(), portIndex);
+ break;
+ }
+ }
+ }
+ }
+
+ public Port[] getPorts() {
+ return ports;
+ }
+
+ @Override
+ public EndpointType getType() {
+ return EndpointType.NETWORK_SWITCH;
+ }
+
+ boolean lookupNetworkTerminal(String terminalName, List<Integer> path) {
+ if (terminalNamePortIndexMap.containsKey(terminalName)) {
+ Integer portIndex = terminalNamePortIndexMap.get(terminalName);
+ path.add(portIndex);
+ NetworkEndpoint endpoint = ports[portIndex.intValue()].getEndpoint();
+ if (endpoint.getType() == EndpointType.NETWORK_SWITCH) {
+ ((NetworkSwitch) endpoint).lookupNetworkTerminal(terminalName, path);
+ }
+ return true;
+ }
+ return false;
+ }
+
+ void getPortList(List<Integer> path, int stepIndex, List<Port> portList) {
+ if (stepIndex >= path.size()) {
+ return;
+ }
+ int portIndex = path.get(stepIndex);
+ Port port = ports[portIndex];
+ portList.add(port);
+ ++stepIndex;
+ if (stepIndex >= path.size()) {
+ return;
+ }
+ NetworkEndpoint endpoint = port.getEndpoint();
+ if (endpoint.getType() != EndpointType.NETWORK_SWITCH) {
+ throw new IllegalArgumentException("Path provided, " + path + ", longer than depth of topology tree");
+ }
+ ((NetworkSwitch) endpoint).getPortList(path, stepIndex, portList);
+ }
+
+ public static class Port implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final NetworkEndpoint endpoint;
+
+ public Port(NetworkEndpoint endpoint) {
+ this.endpoint = endpoint;
+ }
+
+ public NetworkEndpoint getEndpoint() {
+ return endpoint;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/exceptions/AlgebricksException.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/topology/NetworkTerminal.java
similarity index 63%
copy from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/exceptions/AlgebricksException.java
copy to hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/topology/NetworkTerminal.java
index c852ecd..f36da76 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/exceptions/AlgebricksException.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/topology/NetworkTerminal.java
@@ -12,23 +12,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.api.exceptions;
+package edu.uci.ics.hyracks.api.topology;
-public class AlgebricksException extends Exception {
+import java.util.Map;
+
+public class NetworkTerminal extends NetworkEndpoint {
private static final long serialVersionUID = 1L;
- public AlgebricksException() {
+ public NetworkTerminal(String name, Map<String, String> properties) {
+ super(name, properties);
}
- public AlgebricksException(String message) {
- super(message);
- }
-
- public AlgebricksException(Throwable cause) {
- super(cause);
- }
-
- public AlgebricksException(String message, Throwable cause) {
- super(message, cause);
+ @Override
+ public EndpointType getType() {
+ return EndpointType.NETWORK_TERMINAL;
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/topology/TopologyDefinitionParser.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/topology/TopologyDefinitionParser.java
new file mode 100644
index 0000000..0ca6127
--- /dev/null
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/topology/TopologyDefinitionParser.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.topology;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+
+import org.xml.sax.Attributes;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.XMLReader;
+import org.xml.sax.helpers.DefaultHandler;
+import org.xml.sax.helpers.XMLReaderFactory;
+
+import edu.uci.ics.hyracks.api.topology.NetworkEndpoint.EndpointType;
+
+public class TopologyDefinitionParser {
+ private final Stack<ElementStackEntry> stack;
+
+ private boolean inPropertyElement;
+
+ private TopologyDefinitionParser() {
+ stack = new Stack<ElementStackEntry>();
+ inPropertyElement = false;
+ }
+
+ public static ClusterTopology parse(InputSource in) throws IOException, SAXException {
+ TopologyDefinitionParser parser = new TopologyDefinitionParser();
+ return parser.parseInternal(in);
+ }
+
+ private ClusterTopology parseInternal(InputSource in) throws IOException, SAXException {
+ XMLReader parser;
+ parser = XMLReaderFactory.createXMLReader();
+ SAXContentHandler handler = new SAXContentHandler();
+ parser.setContentHandler(handler);
+ parser.parse(in);
+ if (stack.size() != 1) {
+ throw new IllegalStateException("Malformed topology definition");
+ }
+ ElementStackEntry e = stack.pop();
+ if (e.ports.size() != 1) {
+ throw new IllegalArgumentException("Malformed topology definition");
+ }
+ NetworkEndpoint endpoint = e.ports.get(0).getEndpoint();
+ if (endpoint.getType() != EndpointType.NETWORK_SWITCH) {
+ throw new IllegalArgumentException("Top level content in cluster-topology must be network-switch");
+ }
+ return new ClusterTopology((NetworkSwitch) endpoint);
+ }
+
+ private class SAXContentHandler extends DefaultHandler {
+ @Override
+ public void endElement(String uri, String localName, String qName) throws SAXException {
+ if ("network-switch".equals(localName) || "terminal".equals(localName)) {
+ ElementStackEntry e = stack.pop();
+ NetworkEndpoint endpoint = e.type == EndpointType.NETWORK_SWITCH ? new NetworkSwitch(e.name,
+ e.properties, e.ports.toArray(new NetworkSwitch.Port[e.ports.size()])) : new NetworkTerminal(
+ e.name, e.properties);
+ stack.peek().ports.add(new NetworkSwitch.Port(endpoint));
+ } else if ("property".equals(localName)) {
+ if (!inPropertyElement) {
+ throw new IllegalStateException("Improperly nested property element encountered");
+ }
+ inPropertyElement = false;
+ }
+ }
+
+ @Override
+ public void startElement(String uri, String localName, String qName, Attributes atts) throws SAXException {
+ if ("cluster-topology".equals(localName)) {
+ if (!stack.isEmpty()) {
+ throw new IllegalStateException("Encountered unexpected " + qName);
+ }
+ stack.push(new ElementStackEntry(null, ""));
+ } else if ("network-switch".equals(localName) || "terminal".equals(localName)) {
+ String name = atts.getValue("", "name");
+ if (name == null) {
+ throw new IllegalStateException("Encountered " + localName + " element with no name attribute");
+ }
+ EndpointType type = "network-switch".equals(localName) ? EndpointType.NETWORK_SWITCH
+ : EndpointType.NETWORK_TERMINAL;
+ ElementStackEntry e = new ElementStackEntry(type, name);
+ stack.push(e);
+ } else if ("property".equals(localName)) {
+ if (inPropertyElement) {
+ throw new IllegalStateException("Improperly nested property element encountered");
+ }
+ String name = atts.getValue("", "name");
+ if (name == null) {
+ throw new IllegalStateException("Encountered " + localName + " element with no name attribute");
+ }
+ String value = atts.getValue("", "value");
+ if (value == null) {
+ throw new IllegalStateException("Encountered " + localName + " element with no value attribute");
+ }
+ stack.peek().properties.put(name, value);
+ inPropertyElement = true;
+ }
+ }
+ }
+
+ private static class ElementStackEntry {
+ private final EndpointType type;
+
+ private final String name;
+
+ private final Map<String, String> properties;
+
+ private final List<NetworkSwitch.Port> ports;
+
+ public ElementStackEntry(EndpointType type, String name) {
+ this.type = type;
+ this.name = name;
+ this.properties = new HashMap<String, String>();
+ ports = new ArrayList<NetworkSwitch.Port>();
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/ClusterControllerService.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/ClusterControllerService.java
index ebeb8a6..32b031d 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/ClusterControllerService.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/ClusterControllerService.java
@@ -15,6 +15,7 @@
package edu.uci.ics.hyracks.control.cc;
import java.io.File;
+import java.io.FileReader;
import java.net.InetSocketAddress;
import java.util.HashMap;
import java.util.Hashtable;
@@ -24,27 +25,32 @@
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.xml.sax.InputSource;
+
import edu.uci.ics.hyracks.api.client.ClusterControllerInfo;
import edu.uci.ics.hyracks.api.client.HyracksClientInterfaceFunctions;
import edu.uci.ics.hyracks.api.client.NodeControllerInfo;
import edu.uci.ics.hyracks.api.context.ICCContext;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.api.topology.ClusterTopology;
+import edu.uci.ics.hyracks.api.topology.TopologyDefinitionParser;
import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
import edu.uci.ics.hyracks.control.cc.job.JobRun;
import edu.uci.ics.hyracks.control.cc.web.WebServer;
import edu.uci.ics.hyracks.control.cc.work.ApplicationCreateWork;
import edu.uci.ics.hyracks.control.cc.work.ApplicationDestroyWork;
+import edu.uci.ics.hyracks.control.cc.work.ApplicationMessageWork;
import edu.uci.ics.hyracks.control.cc.work.ApplicationStartWork;
import edu.uci.ics.hyracks.control.cc.work.ApplicationStateChangeWork;
import edu.uci.ics.hyracks.control.cc.work.GetIpAddressNodeNameMapWork;
import edu.uci.ics.hyracks.control.cc.work.GetJobStatusWork;
import edu.uci.ics.hyracks.control.cc.work.GetNodeControllersInfoWork;
-import edu.uci.ics.hyracks.control.cc.work.JobCreateWork;
import edu.uci.ics.hyracks.control.cc.work.JobStartWork;
import edu.uci.ics.hyracks.control.cc.work.JobletCleanupNotificationWork;
import edu.uci.ics.hyracks.control.cc.work.NodeHeartbeatWork;
@@ -100,7 +106,7 @@
private final WorkQueue workQueue;
- private final Executor executor;
+ private final ExecutorService executor;
private final Timer timer;
@@ -136,6 +142,7 @@
};
workQueue = new WorkQueue();
this.timer = new Timer(true);
+ final ClusterTopology topology = computeClusterTopology(ccConfig);
ccContext = new ICCContext() {
@Override
public void getIPAddressNodeMap(Map<String, Set<String>> map) throws Exception {
@@ -147,11 +154,29 @@
public ClusterControllerInfo getClusterControllerInfo() {
return info;
}
+
+ @Override
+ public ClusterTopology getClusterTopology() {
+ return topology;
+ }
};
sweeper = new DeadNodeSweeper();
jobCounter = 0;
}
+ private static ClusterTopology computeClusterTopology(CCConfig ccConfig) throws Exception {
+ if (ccConfig.clusterTopologyDefinition == null) {
+ return null;
+ }
+ FileReader fr = new FileReader(ccConfig.clusterTopologyDefinition);
+ InputSource in = new InputSource(fr);
+ try {
+ return TopologyDefinitionParser.parse(in);
+ } finally {
+ fr.close();
+ }
+ }
+
@Override
public void start() throws Exception {
LOGGER.log(Level.INFO, "Starting ClusterControllerService: " + this);
@@ -170,6 +195,7 @@
@Override
public void stop() throws Exception {
LOGGER.log(Level.INFO, "Stopping ClusterControllerService");
+ executor.shutdownNow();
webServer.stop();
sweeper.cancel();
workQueue.stop();
@@ -279,14 +305,6 @@
return;
}
- case CREATE_JOB: {
- HyracksClientInterfaceFunctions.CreateJobFunction cjf = (HyracksClientInterfaceFunctions.CreateJobFunction) fn;
- JobId jobId = createJobId();
- workQueue.schedule(new JobCreateWork(ClusterControllerService.this, jobId, cjf.getAppName(), cjf
- .getJobSpec(), cjf.getJobFlags(), new IPCResponder<JobId>(handle, mid)));
- return;
- }
-
case GET_JOB_STATUS: {
HyracksClientInterfaceFunctions.GetJobStatusFunction gjsf = (HyracksClientInterfaceFunctions.GetJobStatusFunction) fn;
workQueue.schedule(new GetJobStatusWork(ClusterControllerService.this, gjsf.getJobId(),
@@ -296,8 +314,9 @@
case START_JOB: {
HyracksClientInterfaceFunctions.StartJobFunction sjf = (HyracksClientInterfaceFunctions.StartJobFunction) fn;
- workQueue.schedule(new JobStartWork(ClusterControllerService.this, sjf.getJobId(),
- new IPCResponder<Object>(handle, mid)));
+ JobId jobId = createJobId();
+ workQueue.schedule(new JobStartWork(ClusterControllerService.this, sjf.getAppName(), sjf
+ .getACGGFBytes(), sjf.getJobFlags(), jobId, new IPCResponder<JobId>(handle, mid)));
return;
}
@@ -313,6 +332,15 @@
new IPCResponder<Map<String, NodeControllerInfo>>(handle, mid)));
return;
}
+
+ case GET_CLUSTER_TOPOLOGY: {
+ try {
+ handle.send(mid, ccContext.getClusterTopology(), null);
+ } catch (IPCException e) {
+ e.printStackTrace();
+ }
+ return;
+ }
}
try {
handle.send(mid, null, new IllegalArgumentException("Unknown function " + fn.getFunctionId()));
@@ -391,6 +419,12 @@
workQueue.schedule(new ApplicationStateChangeWork(ClusterControllerService.this, astrf));
return;
}
+ case SEND_APPLICATION_MESSAGE: {
+ CCNCFunctions.SendApplicationMessageFunction rsf = (CCNCFunctions.SendApplicationMessageFunction) fn;
+ workQueue.schedule(new ApplicationMessageWork(ClusterControllerService.this, rsf.getMessage(), rsf
+ .getAppName(), rsf.getNodeId()));
+ return;
+ }
}
LOGGER.warning("Unknown function: " + fn.getFunctionId());
}
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.java
index 1d96a71..e165415 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.java
@@ -33,9 +33,8 @@
import edu.uci.ics.hyracks.api.dataflow.TaskId;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.work.GetJobActivityGraphJSONWork;
+import edu.uci.ics.hyracks.control.cc.work.GetActivityClusterGraphJSONWork;
import edu.uci.ics.hyracks.control.cc.work.GetJobRunJSONWork;
-import edu.uci.ics.hyracks.control.cc.work.GetJobSpecificationJSONWork;
public class JobDetailsPage extends AbstractPage {
private static final long serialVersionUID = 1L;
@@ -49,19 +48,13 @@
JobId jobId = JobId.parse(jobIdStr.toString());
- GetJobSpecificationJSONWork gjsw = new GetJobSpecificationJSONWork(ccs, jobId);
- ccs.getWorkQueue().scheduleAndSync(gjsw);
- Label jobspec = new Label("job-specification", gjsw.getJSON().toString());
- jobspec.setEscapeModelStrings(false);
- add(jobspec);
-
- GetJobActivityGraphJSONWork gjagw = new GetJobActivityGraphJSONWork(ccs, jobId);
- ccs.getWorkQueue().scheduleAndSync(gjagw);
- Label jag = new Label("job-activity-graph", gjagw.getJSON().toString());
+ GetActivityClusterGraphJSONWork gacgw = new GetActivityClusterGraphJSONWork(ccs, jobId);
+ ccs.getWorkQueue().scheduleAndSync(gacgw);
+ Label jag = new Label("activity-cluster-graph", gacgw.getJSON().toString());
jag.setEscapeModelStrings(false);
add(jag);
- JSONObject jagO = gjagw.getJSON();
+ JSONObject jagO = gacgw.getJSON();
Map<ActivityId, String> activityMap = new HashMap<ActivityId, String>();
if (jagO.has("activities")) {
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/application/CCApplicationContext.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/application/CCApplicationContext.java
index 28f7c59..24dfa7c 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/application/CCApplicationContext.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/application/CCApplicationContext.java
@@ -25,11 +25,10 @@
import edu.uci.ics.hyracks.api.application.ICCBootstrap;
import edu.uci.ics.hyracks.api.context.ICCContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.job.IActivityClusterGraphGeneratorFactory;
import edu.uci.ics.hyracks.api.job.IJobLifecycleListener;
-import edu.uci.ics.hyracks.api.job.IJobSpecificationFactory;
import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.control.cc.job.DeserializingJobSpecificationFactory;
+import edu.uci.ics.hyracks.api.util.JavaSerializationUtils;
import edu.uci.ics.hyracks.control.common.application.ApplicationContext;
import edu.uci.ics.hyracks.control.common.context.ServerContext;
import edu.uci.ics.hyracks.control.common.work.IResultCallback;
@@ -43,8 +42,6 @@
protected IResultCallback<Object> initializationCallback;
protected IResultCallback<Object> deinitializationCallback;
- private IJobSpecificationFactory jobSpecFactory;
-
private List<IJobLifecycleListener> jobLifecycleListeners;
public CCApplicationContext(ServerContext serverCtx, ICCContext ccContext, String appName) throws IOException {
@@ -52,7 +49,6 @@
this.ccContext = ccContext;
initPendingNodeIds = new HashSet<String>();
deinitPendingNodeIds = new HashSet<String>();
- jobSpecFactory = DeserializingJobSpecificationFactory.INSTANCE;
jobLifecycleListeners = new ArrayList<IJobLifecycleListener>();
}
@@ -66,13 +62,15 @@
return ccContext;
}
- @Override
- public void setJobSpecificationFactory(IJobSpecificationFactory jobSpecFactory) {
- this.jobSpecFactory = jobSpecFactory;
- }
-
- public JobSpecification createJobSpecification(byte[] bytes) throws HyracksException {
- return jobSpecFactory.createJobSpecification(bytes, (ICCBootstrap) bootstrap, this);
+ public IActivityClusterGraphGeneratorFactory createActivityClusterGraphGeneratorFactory(byte[] bytes)
+ throws HyracksException {
+ try {
+ return (IActivityClusterGraphGeneratorFactory) JavaSerializationUtils.deserialize(bytes, getClassLoader());
+ } catch (IOException e) {
+ throw new HyracksException(e);
+ } catch (ClassNotFoundException e) {
+ throw new HyracksException(e);
+ }
}
@Override
@@ -104,9 +102,10 @@
}
}
- public synchronized void notifyJobCreation(JobId jobId, JobSpecification specification) throws HyracksException {
+ public synchronized void notifyJobCreation(JobId jobId, IActivityClusterGraphGeneratorFactory acggf)
+ throws HyracksException {
for (IJobLifecycleListener l : jobLifecycleListeners) {
- l.notifyJobCreation(jobId, specification);
+ l.notifyJobCreation(jobId, acggf);
}
}
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/ActivityCluster.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/ActivityCluster.java
deleted file mode 100644
index 13db627..0000000
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/ActivityCluster.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import edu.uci.ics.hyracks.api.dataflow.ActivityId;
-
-public class ActivityCluster {
- private final JobRun jobRun;
-
- private final Set<ActivityId> activities;
-
- private final Set<ActivityCluster> dependencies;
-
- private final Set<ActivityCluster> dependents;
-
- private ActivityClusterId id;
-
- private ActivityClusterPlan acp;
-
- public ActivityCluster(JobRun jobRun, Set<ActivityId> activities) {
- this.jobRun = jobRun;
- this.activities = activities;
- dependencies = new HashSet<ActivityCluster>();
- dependents = new HashSet<ActivityCluster>();
- }
-
- public Set<ActivityId> getActivities() {
- return activities;
- }
-
- public void addDependency(ActivityCluster stage) {
- dependencies.add(stage);
- }
-
- public void addDependent(ActivityCluster stage) {
- dependents.add(stage);
- }
-
- public Set<ActivityCluster> getDependencies() {
- return dependencies;
- }
-
- public Set<ActivityCluster> getDependents() {
- return dependents;
- }
-
- public JobRun getJobRun() {
- return jobRun;
- }
-
- public ActivityClusterId getActivityClusterId() {
- return id;
- }
-
- public void setActivityClusterId(ActivityClusterId id) {
- this.id = id;
- }
-
- public int getMaxTaskClusterReattempts() {
- return jobRun.getJobActivityGraph().getJobSpecification().getMaxReattempts();
- }
-
- public ActivityClusterPlan getPlan() {
- return acp;
- }
-
- public void setPlan(ActivityClusterPlan acp) {
- this.acp = acp;
- }
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/DeserializingJobSpecificationFactory.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/DeserializingJobSpecificationFactory.java
deleted file mode 100644
index c746f34..0000000
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/DeserializingJobSpecificationFactory.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job;
-
-import java.io.IOException;
-
-import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
-import edu.uci.ics.hyracks.api.application.ICCBootstrap;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.job.IJobSpecificationFactory;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.api.util.JavaSerializationUtils;
-
-public class DeserializingJobSpecificationFactory implements IJobSpecificationFactory {
- public static final IJobSpecificationFactory INSTANCE = new DeserializingJobSpecificationFactory();
-
- private DeserializingJobSpecificationFactory() {
- }
-
- @Override
- public JobSpecification createJobSpecification(byte[] bytes, ICCBootstrap bootstrap, ICCApplicationContext appCtx)
- throws HyracksException {
- try {
- return (JobSpecification) JavaSerializationUtils.deserialize(bytes, appCtx.getClassLoader());
- } catch (IOException e) {
- throw new HyracksException(e);
- } catch (ClassNotFoundException e) {
- throw new HyracksException(e);
- }
- }
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/JobActivityGraphBuilder.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/JobActivityGraphBuilder.java
deleted file mode 100644
index 6effd84..0000000
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/JobActivityGraphBuilder.java
+++ /dev/null
@@ -1,92 +0,0 @@
-package edu.uci.ics.hyracks.control.cc.job;
-
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import edu.uci.ics.hyracks.api.dataflow.ActivityId;
-import edu.uci.ics.hyracks.api.dataflow.IActivity;
-import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder;
-import edu.uci.ics.hyracks.api.job.JobActivityGraph;
-import edu.uci.ics.hyracks.api.job.JobFlag;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public class JobActivityGraphBuilder implements IActivityGraphBuilder {
- private static final Logger LOGGER = Logger.getLogger(JobActivityGraphBuilder.class.getName());
-
- private JobActivityGraph jag;
-
- @Override
- public void addBlockingEdge(IActivity blocker, IActivity blocked) {
- addToValueSet(jag.getBlocker2BlockedMap(), blocker.getActivityId(), blocked.getActivityId());
- addToValueSet(jag.getBlocked2BlockerMap(), blocked.getActivityId(), blocker.getActivityId());
- }
-
- @Override
- public void addSourceEdge(int operatorInputIndex, IActivity task, int taskInputIndex) {
- if (LOGGER.isLoggable(Level.FINEST)) {
- LOGGER.finest("Adding source edge: " + task.getActivityId().getOperatorDescriptorId() + ":"
- + operatorInputIndex + " -> " + task.getActivityId() + ":" + taskInputIndex);
- }
- insertIntoIndexedMap(jag.getActivityInputMap(), task.getActivityId(), taskInputIndex, operatorInputIndex);
- insertIntoIndexedMap(jag.getOperatorInputMap(), task.getActivityId().getOperatorDescriptorId(),
- operatorInputIndex, task.getActivityId());
- }
-
- @Override
- public void addTargetEdge(int operatorOutputIndex, IActivity task, int taskOutputIndex) {
- if (LOGGER.isLoggable(Level.FINEST)) {
- LOGGER.finest("Adding target edge: " + task.getActivityId().getOperatorDescriptorId() + ":"
- + operatorOutputIndex + " -> " + task.getActivityId() + ":" + taskOutputIndex);
- }
- insertIntoIndexedMap(jag.getActivityOutputMap(), task.getActivityId(), taskOutputIndex, operatorOutputIndex);
- insertIntoIndexedMap(jag.getOperatorOutputMap(), task.getActivityId().getOperatorDescriptorId(),
- operatorOutputIndex, task.getActivityId());
- }
-
- @Override
- public void addActivity(IActivity task) {
- ActivityId activityId = task.getActivityId();
- jag.getActivityNodeMap().put(activityId, task);
- addToValueSet(jag.getOperatorActivityMap(), activityId.getOperatorDescriptorId(), activityId);
- }
-
- private <K, V> void addToValueSet(Map<K, Set<V>> map, K n1, V n2) {
- Set<V> targets = map.get(n1);
- if (targets == null) {
- targets = new HashSet<V>();
- map.put(n1, targets);
- }
- targets.add(n2);
- }
-
- private <T> void extend(List<T> list, int index) {
- int n = list.size();
- for (int i = n; i <= index; ++i) {
- list.add(null);
- }
- }
-
- public void init(String appName, JobSpecification jobSpec, EnumSet<JobFlag> jobFlags) {
- jag = new JobActivityGraph(appName, jobSpec, jobFlags);
- }
-
- private <K, V> void insertIntoIndexedMap(Map<K, List<V>> map, K key, int index, V value) {
- List<V> vList = map.get(key);
- if (vList == null) {
- vList = new ArrayList<V>();
- map.put(key, vList);
- }
- extend(vList, index);
- vList.set(index, value);
- }
-
- public JobActivityGraph getActivityGraph() {
- return jag;
- }
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/JobRun.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/JobRun.java
index 707ef1f..6f26de2 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/JobRun.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/JobRun.java
@@ -14,6 +14,7 @@
*/
package edu.uci.ics.hyracks.control.cc.job;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -29,10 +30,15 @@
import edu.uci.ics.hyracks.api.dataflow.TaskId;
import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.job.JobActivityGraph;
+import edu.uci.ics.hyracks.api.job.ActivityCluster;
+import edu.uci.ics.hyracks.api.job.ActivityClusterGraph;
+import edu.uci.ics.hyracks.api.job.ActivityClusterId;
+import edu.uci.ics.hyracks.api.job.IActivityClusterGraphGenerator;
+import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobStatus;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
import edu.uci.ics.hyracks.control.cc.partitions.PartitionMatchMaker;
import edu.uci.ics.hyracks.control.cc.scheduler.ActivityPartitionDetails;
import edu.uci.ics.hyracks.control.cc.scheduler.JobScheduler;
@@ -41,7 +47,17 @@
public class JobRun implements IJobStatusConditionVariable {
private final JobId jobId;
- private final JobActivityGraph jag;
+ private final String applicationName;
+
+ private final IActivityClusterGraphGenerator acgg;
+
+ private final ActivityClusterGraph acg;
+
+ private final JobScheduler scheduler;
+
+ private final EnumSet<JobFlag> jobFlags;
+
+ private final Map<ActivityClusterId, ActivityClusterPlan> activityClusterPlanMap;
private final PartitionMatchMaker pmm;
@@ -51,14 +67,8 @@
private final JobProfile profile;
- private Set<ActivityCluster> activityClusters;
-
- private final Map<ActivityId, ActivityCluster> activityClusterMap;
-
private final Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicyMap;
- private JobScheduler js;
-
private long createTime;
private long startTime;
@@ -73,14 +83,19 @@
private Exception pendingException;
- public JobRun(JobId jobId, JobActivityGraph plan) {
+ public JobRun(ClusterControllerService ccs, JobId jobId, String applicationName,
+ IActivityClusterGraphGenerator acgg, EnumSet<JobFlag> jobFlags) {
this.jobId = jobId;
- this.jag = plan;
+ this.applicationName = applicationName;
+ this.acgg = acgg;
+ this.acg = acgg.initialize();
+ this.scheduler = new JobScheduler(ccs, this, acgg.getConstraints());
+ this.jobFlags = jobFlags;
+ activityClusterPlanMap = new HashMap<ActivityClusterId, ActivityClusterPlan>();
pmm = new PartitionMatchMaker();
participatingNodeIds = new HashSet<String>();
cleanupPendingNodeIds = new HashSet<String>();
profile = new JobProfile(jobId);
- activityClusterMap = new HashMap<ActivityId, ActivityCluster>();
connectorPolicyMap = new HashMap<ConnectorDescriptorId, IConnectorPolicy>();
}
@@ -88,8 +103,20 @@
return jobId;
}
- public JobActivityGraph getJobActivityGraph() {
- return jag;
+ public String getApplicationName() {
+ return applicationName;
+ }
+
+ public ActivityClusterGraph getActivityClusterGraph() {
+ return acg;
+ }
+
+ public EnumSet<JobFlag> getFlags() {
+ return jobFlags;
+ }
+
+ public Map<ActivityClusterId, ActivityClusterPlan> getActivityClusterPlanMap() {
+ return activityClusterPlanMap;
}
public PartitionMatchMaker getPartitionMatchMaker() {
@@ -169,24 +196,8 @@
return profile;
}
- public void setScheduler(JobScheduler js) {
- this.js = js;
- }
-
public JobScheduler getScheduler() {
- return js;
- }
-
- public Map<ActivityId, ActivityCluster> getActivityClusterMap() {
- return activityClusterMap;
- }
-
- public Set<ActivityCluster> getActivityClusters() {
- return activityClusters;
- }
-
- public void setActivityClusters(Set<ActivityCluster> activityClusters) {
- this.activityClusters = activityClusters;
+ return scheduler;
}
public Map<ConnectorDescriptorId, IConnectorPolicy> getConnectorPolicyMap() {
@@ -197,37 +208,31 @@
JSONObject result = new JSONObject();
result.put("job-id", jobId.toString());
- result.put("application-name", jag.getApplicationName());
+ result.put("application-name", applicationName);
result.put("status", getStatus());
result.put("create-time", getCreateTime());
result.put("start-time", getCreateTime());
result.put("end-time", getCreateTime());
JSONArray aClusters = new JSONArray();
- for (ActivityCluster ac : activityClusters) {
+ for (ActivityCluster ac : acg.getActivityClusterMap().values()) {
JSONObject acJSON = new JSONObject();
- acJSON.put("activity-cluster-id", String.valueOf(ac.getActivityClusterId()));
+ acJSON.put("activity-cluster-id", String.valueOf(ac.getId()));
JSONArray activitiesJSON = new JSONArray();
- for (ActivityId aid : ac.getActivities()) {
+ for (ActivityId aid : ac.getActivityMap().keySet()) {
activitiesJSON.put(aid);
}
acJSON.put("activities", activitiesJSON);
- JSONArray dependentsJSON = new JSONArray();
- for (ActivityCluster dependent : ac.getDependents()) {
- dependentsJSON.put(String.valueOf(dependent.getActivityClusterId()));
- }
- acJSON.put("dependents", dependentsJSON);
-
JSONArray dependenciesJSON = new JSONArray();
for (ActivityCluster dependency : ac.getDependencies()) {
- dependenciesJSON.put(String.valueOf(dependency.getActivityClusterId()));
+ dependenciesJSON.put(String.valueOf(dependency.getId()));
}
acJSON.put("dependencies", dependenciesJSON);
- ActivityClusterPlan acp = ac.getPlan();
+ ActivityClusterPlan acp = activityClusterPlanMap.get(ac.getId());
if (acp == null) {
acJSON.put("plan", (Object) null);
} else {
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskCluster.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskCluster.java
index f41cda9..17beb2d 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskCluster.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskCluster.java
@@ -20,6 +20,7 @@
import java.util.List;
import java.util.Set;
+import edu.uci.ics.hyracks.api.job.ActivityCluster;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
public class TaskCluster {
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskClusterId.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskClusterId.java
index b1000dd..1218c14 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskClusterId.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskClusterId.java
@@ -16,6 +16,8 @@
import java.io.Serializable;
+import edu.uci.ics.hyracks.api.job.ActivityClusterId;
+
public final class TaskClusterId implements Serializable {
private static final long serialVersionUID = 1L;
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterGraphBuilder.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterGraphBuilder.java
deleted file mode 100644
index f407ae3..0000000
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterGraphBuilder.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.scheduler;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.commons.lang3.tuple.Pair;
-
-import edu.uci.ics.hyracks.api.dataflow.ActivityId;
-import edu.uci.ics.hyracks.api.dataflow.IActivity;
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
-import edu.uci.ics.hyracks.api.job.JobActivityGraph;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
-import edu.uci.ics.hyracks.control.cc.job.ActivityClusterId;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-
-public class ActivityClusterGraphBuilder {
- private static final Logger LOGGER = Logger.getLogger(ActivityClusterGraphBuilder.class.getName());
-
- private final JobRun jobRun;
-
- public ActivityClusterGraphBuilder(JobRun jobRun) {
- this.jobRun = jobRun;
- }
-
- private static Pair<ActivityId, ActivityId> findMergePair(JobActivityGraph jag, JobSpecification spec,
- Set<ActivityCluster> eqSets) {
- Map<ActivityId, IActivity> activityNodeMap = jag.getActivityNodeMap();
- for (ActivityCluster eqSet : eqSets) {
- for (ActivityId t : eqSet.getActivities()) {
- IActivity activity = activityNodeMap.get(t);
- List<Integer> inputList = jag.getActivityInputMap().get(t);
- if (inputList != null) {
- for (Integer idx : inputList) {
- IConnectorDescriptor conn = spec.getInputConnectorDescriptor(activity.getActivityId()
- .getOperatorDescriptorId(), idx);
- OperatorDescriptorId producerId = spec.getProducer(conn).getOperatorId();
- int producerOutputIndex = spec.getProducerOutputIndex(conn);
- ActivityId inTask = jag.getOperatorOutputMap().get(producerId).get(producerOutputIndex);
- if (!eqSet.getActivities().contains(inTask)) {
- return Pair.<ActivityId, ActivityId> of(t, inTask);
- }
- }
- }
- List<Integer> outputList = jag.getActivityOutputMap().get(t);
- if (outputList != null) {
- for (Integer idx : outputList) {
- IConnectorDescriptor conn = spec.getOutputConnectorDescriptor(activity.getActivityId()
- .getOperatorDescriptorId(), idx);
- OperatorDescriptorId consumerId = spec.getConsumer(conn).getOperatorId();
- int consumerInputIndex = spec.getConsumerInputIndex(conn);
- ActivityId outTask = jag.getOperatorInputMap().get(consumerId).get(consumerInputIndex);
- if (!eqSet.getActivities().contains(outTask)) {
- return Pair.<ActivityId, ActivityId> of(t, outTask);
- }
- }
- }
- }
- }
- return null;
- }
-
- public Set<ActivityCluster> inferActivityClusters(JobActivityGraph jag) {
- JobSpecification spec = jag.getJobSpecification();
-
- /*
- * Build initial equivalence sets map. We create a map such that for each IOperatorTask, t -> { t }
- */
- Map<ActivityId, ActivityCluster> stageMap = new HashMap<ActivityId, ActivityCluster>();
- Set<ActivityCluster> stages = new HashSet<ActivityCluster>();
- for (Set<ActivityId> taskIds : jag.getOperatorActivityMap().values()) {
- for (ActivityId taskId : taskIds) {
- Set<ActivityId> eqSet = new HashSet<ActivityId>();
- eqSet.add(taskId);
- ActivityCluster stage = new ActivityCluster(jobRun, eqSet);
- stageMap.put(taskId, stage);
- stages.add(stage);
- }
- }
-
- boolean changed = true;
- while (changed) {
- changed = false;
- Pair<ActivityId, ActivityId> pair = findMergePair(jag, spec, stages);
- if (pair != null) {
- merge(stageMap, stages, pair.getLeft(), pair.getRight());
- changed = true;
- }
- }
-
- Map<ActivityId, Set<ActivityId>> blocker2BlockedMap = jag.getBlocker2BlockedMap();
- for (ActivityCluster s : stages) {
- Set<ActivityCluster> blockedStages = new HashSet<ActivityCluster>();
- for (ActivityId t : s.getActivities()) {
- Set<ActivityId> blockedTasks = blocker2BlockedMap.get(t);
- if (blockedTasks != null) {
- for (ActivityId bt : blockedTasks) {
- blockedStages.add(stageMap.get(bt));
- }
- }
- }
- for (ActivityCluster bs : blockedStages) {
- bs.addDependency(s);
- s.addDependent(bs);
- }
- }
- Set<ActivityCluster> roots = new HashSet<ActivityCluster>();
- int idCounter = 0;
- for (ActivityCluster s : stages) {
- s.setActivityClusterId(new ActivityClusterId(idCounter++));
- if (s.getDependents().isEmpty()) {
- roots.add(s);
- }
- }
- jobRun.setActivityClusters(stages);
- jobRun.getActivityClusterMap().putAll(stageMap);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Inferred " + stages.size() + " stages");
- for (ActivityCluster s : stages) {
- LOGGER.info(s.toString());
- }
- }
- return roots;
- }
-
- private void merge(Map<ActivityId, ActivityCluster> eqSetMap, Set<ActivityCluster> eqSets, ActivityId t1,
- ActivityId t2) {
- ActivityCluster stage1 = eqSetMap.get(t1);
- Set<ActivityId> s1 = stage1.getActivities();
- ActivityCluster stage2 = eqSetMap.get(t2);
- Set<ActivityId> s2 = stage2.getActivities();
-
- Set<ActivityId> mergedSet = new HashSet<ActivityId>();
- mergedSet.addAll(s1);
- mergedSet.addAll(s2);
-
- eqSets.remove(stage1);
- eqSets.remove(stage2);
- ActivityCluster mergedStage = new ActivityCluster(jobRun, mergedSet);
- eqSets.add(mergedStage);
-
- for (ActivityId t : mergedSet) {
- eqSetMap.put(t, mergedStage);
- }
- }
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterPlanner.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterPlanner.java
index e6e6c2f..6c52aac 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterPlanner.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterPlanner.java
@@ -38,9 +38,9 @@
import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicyAssignmentPolicy;
import edu.uci.ics.hyracks.api.dataflow.connectors.PipeliningConnectorPolicy;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.job.JobActivityGraph;
+import edu.uci.ics.hyracks.api.job.ActivityCluster;
+import edu.uci.ics.hyracks.api.job.ActivityClusterGraph;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
import edu.uci.ics.hyracks.control.cc.job.ActivityClusterPlan;
import edu.uci.ics.hyracks.control.cc.job.ActivityPlan;
import edu.uci.ics.hyracks.control.cc.job.JobRun;
@@ -62,7 +62,7 @@
partitionProducingTaskClusterMap = new HashMap<PartitionId, TaskCluster>();
}
- public void planActivityCluster(ActivityCluster ac) throws HyracksException {
+ public ActivityClusterPlan planActivityCluster(ActivityCluster ac) throws HyracksException {
JobRun jobRun = scheduler.getJobRun();
Map<ActivityId, ActivityPartitionDetails> pcMap = computePartitionCounts(ac);
@@ -80,16 +80,16 @@
}
}
- ac.setPlan(new ActivityClusterPlan(taskClusters, activityPlanMap));
+ return new ActivityClusterPlan(taskClusters, activityPlanMap);
}
private Map<ActivityId, ActivityPlan> buildActivityPlanMap(ActivityCluster ac, JobRun jobRun,
Map<ActivityId, ActivityPartitionDetails> pcMap) {
Map<ActivityId, ActivityPlan> activityPlanMap = new HashMap<ActivityId, ActivityPlan>();
Set<ActivityId> depAnIds = new HashSet<ActivityId>();
- for (ActivityId anId : ac.getActivities()) {
+ for (ActivityId anId : ac.getActivityMap().keySet()) {
depAnIds.clear();
- getDependencyActivityIds(depAnIds, anId);
+ getDependencyActivityIds(depAnIds, anId, ac);
ActivityPartitionDetails apd = pcMap.get(anId);
Task[] tasks = new Task[apd.getPartitionCount()];
ActivityPlan activityPlan = new ActivityPlan(apd);
@@ -97,8 +97,8 @@
TaskId tid = new TaskId(anId, i);
tasks[i] = new Task(tid, activityPlan);
for (ActivityId danId : depAnIds) {
- ActivityCluster dAC = jobRun.getActivityClusterMap().get(danId);
- ActivityClusterPlan dACP = dAC.getPlan();
+ ActivityCluster dAC = ac.getActivityClusterGraph().getActivityMap().get(danId);
+ ActivityClusterPlan dACP = jobRun.getActivityClusterPlanMap().get(dAC.getId());
assert dACP != null : "IllegalStateEncountered: Dependent AC is being planned without a plan for dependency AC: Encountered no plan for ActivityID "
+ danId;
Task[] dATasks = dACP.getActivityPlanMap().get(danId).getTasks();
@@ -120,7 +120,7 @@
private TaskCluster[] computeTaskClusters(ActivityCluster ac, JobRun jobRun,
Map<ActivityId, ActivityPlan> activityPlanMap) {
- Set<ActivityId> activities = ac.getActivities();
+ Set<ActivityId> activities = ac.getActivityMap().keySet();
Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity = computeTaskConnectivity(jobRun,
activityPlanMap, activities);
@@ -161,15 +161,15 @@
private TaskCluster[] buildConnectorPolicyUnawareTaskClusters(ActivityCluster ac,
Map<ActivityId, ActivityPlan> activityPlanMap) {
List<Task> taskStates = new ArrayList<Task>();
- for (ActivityId anId : ac.getActivities()) {
+ for (ActivityId anId : ac.getActivityMap().keySet()) {
ActivityPlan ap = activityPlanMap.get(anId);
Task[] tasks = ap.getTasks();
for (Task t : tasks) {
taskStates.add(t);
}
}
- TaskCluster tc = new TaskCluster(new TaskClusterId(ac.getActivityClusterId(), 0), ac,
- taskStates.toArray(new Task[taskStates.size()]));
+ TaskCluster tc = new TaskCluster(new TaskClusterId(ac.getId(), 0), ac, taskStates.toArray(new Task[taskStates
+ .size()]));
for (Task t : tc.getTasks()) {
t.setTaskCluster(tc);
}
@@ -179,16 +179,17 @@
private Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> computeTaskConnectivity(JobRun jobRun,
Map<ActivityId, ActivityPlan> activityPlanMap, Set<ActivityId> activities) {
Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity = new HashMap<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>>();
- JobActivityGraph jag = jobRun.getJobActivityGraph();
+ ActivityClusterGraph acg = jobRun.getActivityClusterGraph();
BitSet targetBitmap = new BitSet();
for (ActivityId ac1 : activities) {
+ ActivityCluster ac = acg.getActivityMap().get(ac1);
Task[] ac1TaskStates = activityPlanMap.get(ac1).getTasks();
int nProducers = ac1TaskStates.length;
- List<IConnectorDescriptor> outputConns = jag.getActivityOutputConnectorDescriptors(ac1);
+ List<IConnectorDescriptor> outputConns = ac.getActivityOutputMap().get(ac1);
if (outputConns != null) {
for (IConnectorDescriptor c : outputConns) {
ConnectorDescriptorId cdId = c.getConnectorId();
- ActivityId ac2 = jag.getConsumerActivity(cdId);
+ ActivityId ac2 = ac.getConsumerActivity(cdId);
Task[] ac2TaskStates = activityPlanMap.get(ac2).getTasks();
int nConsumers = ac2TaskStates.length;
for (int i = 0; i < nProducers; ++i) {
@@ -214,7 +215,7 @@
Map<ActivityId, ActivityPlan> activityPlanMap,
Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity) {
Map<TaskId, Set<TaskId>> taskClusterMap = new HashMap<TaskId, Set<TaskId>>();
- for (ActivityId anId : ac.getActivities()) {
+ for (ActivityId anId : ac.getActivityMap().keySet()) {
ActivityPlan ap = activityPlanMap.get(anId);
Task[] tasks = ap.getTasks();
for (Task t : tasks) {
@@ -225,7 +226,7 @@
}
}
- JobRun jobRun = ac.getJobRun();
+ JobRun jobRun = scheduler.getJobRun();
Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies = jobRun.getConnectorPolicyMap();
for (Map.Entry<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> e : taskConnectivity.entrySet()) {
Set<TaskId> cluster = taskClusterMap.get(e.getKey());
@@ -298,7 +299,7 @@
for (TaskId tid : cluster) {
taskStates.add(activityPlanMap.get(tid.getActivityId()).getTasks()[tid.getPartition()]);
}
- TaskCluster tc = new TaskCluster(new TaskClusterId(ac.getActivityClusterId(), counter++), ac,
+ TaskCluster tc = new TaskCluster(new TaskClusterId(ac.getId(), counter++), ac,
taskStates.toArray(new Task[taskStates.size()]));
tcSet.add(tc);
for (TaskId tid : cluster) {
@@ -310,35 +311,34 @@
}
private TaskCluster getTaskCluster(TaskId tid) {
- ActivityCluster ac = scheduler.getJobRun().getActivityClusterMap().get(tid.getActivityId());
- ActivityClusterPlan acp = ac.getPlan();
+ JobRun run = scheduler.getJobRun();
+ ActivityCluster ac = run.getActivityClusterGraph().getActivityMap().get(tid.getActivityId());
+ ActivityClusterPlan acp = run.getActivityClusterPlanMap().get(ac.getId());
Task[] tasks = acp.getActivityPlanMap().get(tid.getActivityId()).getTasks();
Task task = tasks[tid.getPartition()];
assert task.getTaskId().equals(tid);
return task.getTaskCluster();
}
- private void getDependencyActivityIds(Set<ActivityId> depAnIds, ActivityId anId) {
- JobActivityGraph jag = scheduler.getJobRun().getJobActivityGraph();
- Set<ActivityId> blockers = jag.getBlocked2BlockerMap().get(anId);
+ private void getDependencyActivityIds(Set<ActivityId> depAnIds, ActivityId anId, ActivityCluster ac) {
+ Set<ActivityId> blockers = ac.getBlocked2BlockerMap().get(anId);
if (blockers != null) {
depAnIds.addAll(blockers);
}
}
private void assignConnectorPolicy(ActivityCluster ac, Map<ActivityId, ActivityPlan> taskMap) {
- JobActivityGraph jag = scheduler.getJobRun().getJobActivityGraph();
Map<ConnectorDescriptorId, IConnectorPolicy> cPolicyMap = new HashMap<ConnectorDescriptorId, IConnectorPolicy>();
- Set<ActivityId> activities = ac.getActivities();
+ Set<ActivityId> activities = ac.getActivityMap().keySet();
BitSet targetBitmap = new BitSet();
for (ActivityId a1 : activities) {
Task[] ac1TaskStates = taskMap.get(a1).getTasks();
int nProducers = ac1TaskStates.length;
- List<IConnectorDescriptor> outputConns = jag.getActivityOutputConnectorDescriptors(a1);
+ List<IConnectorDescriptor> outputConns = ac.getActivityOutputMap().get(a1);
if (outputConns != null) {
for (IConnectorDescriptor c : outputConns) {
ConnectorDescriptorId cdId = c.getConnectorId();
- ActivityId a2 = jag.getConsumerActivity(cdId);
+ ActivityId a2 = ac.getConsumerActivity(cdId);
Task[] ac2TaskStates = taskMap.get(a2).getTasks();
int nConsumers = ac2TaskStates.length;
@@ -347,17 +347,17 @@
c.indicateTargetPartitions(nProducers, nConsumers, i, targetBitmap);
fanouts[i] = targetBitmap.cardinality();
}
- IConnectorPolicy cp = assignConnectorPolicy(c, nProducers, nConsumers, fanouts);
+ IConnectorPolicy cp = assignConnectorPolicy(ac, c, nProducers, nConsumers, fanouts);
cPolicyMap.put(cdId, cp);
}
}
}
- ac.getJobRun().getConnectorPolicyMap().putAll(cPolicyMap);
+ scheduler.getJobRun().getConnectorPolicyMap().putAll(cPolicyMap);
}
- private IConnectorPolicy assignConnectorPolicy(IConnectorDescriptor c, int nProducers, int nConsumers, int[] fanouts) {
- IConnectorPolicyAssignmentPolicy cpap = scheduler.getJobRun().getJobActivityGraph().getJobSpecification()
- .getConnectorPolicyAssignmentPolicy();
+ private IConnectorPolicy assignConnectorPolicy(ActivityCluster ac, IConnectorDescriptor c, int nProducers,
+ int nConsumers, int[] fanouts) {
+ IConnectorPolicyAssignmentPolicy cpap = ac.getConnectorPolicyAssignmentPolicy();
if (cpap != null) {
return cpap.getConnectorPolicyAssignment(c, nProducers, nConsumers, fanouts);
}
@@ -367,9 +367,8 @@
private Map<ActivityId, ActivityPartitionDetails> computePartitionCounts(ActivityCluster ac)
throws HyracksException {
PartitionConstraintSolver solver = scheduler.getSolver();
- JobRun jobRun = scheduler.getJobRun();
Set<LValueConstraintExpression> lValues = new HashSet<LValueConstraintExpression>();
- for (ActivityId anId : ac.getActivities()) {
+ for (ActivityId anId : ac.getActivityMap().keySet()) {
lValues.add(new PartitionCountExpression(anId.getOperatorDescriptorId()));
}
solver.solve(lValues);
@@ -390,25 +389,28 @@
nPartMap.put(((PartitionCountExpression) lv).getOperatorDescriptorId(), Integer.valueOf(nParts));
}
Map<ActivityId, ActivityPartitionDetails> activityPartsMap = new HashMap<ActivityId, ActivityPartitionDetails>();
- for (ActivityId anId : ac.getActivities()) {
+ for (ActivityId anId : ac.getActivityMap().keySet()) {
int nParts = nPartMap.get(anId.getOperatorDescriptorId());
int[] nInputPartitions = null;
- List<IConnectorDescriptor> inputs = jobRun.getJobActivityGraph().getActivityInputConnectorDescriptors(anId);
+ List<IConnectorDescriptor> inputs = ac.getActivityInputMap().get(anId);
if (inputs != null) {
nInputPartitions = new int[inputs.size()];
for (int i = 0; i < nInputPartitions.length; ++i) {
- nInputPartitions[i] = nPartMap.get(jobRun.getJobActivityGraph()
- .getProducerActivity(inputs.get(i).getConnectorId()).getOperatorDescriptorId());
+ ConnectorDescriptorId cdId = inputs.get(i).getConnectorId();
+ ActivityId aid = ac.getProducerActivity(cdId);
+ Integer nPartInt = nPartMap.get(aid.getOperatorDescriptorId());
+ nInputPartitions[i] = nPartInt;
}
}
int[] nOutputPartitions = null;
- List<IConnectorDescriptor> outputs = jobRun.getJobActivityGraph().getActivityOutputConnectorDescriptors(
- anId);
+ List<IConnectorDescriptor> outputs = ac.getActivityOutputMap().get(anId);
if (outputs != null) {
nOutputPartitions = new int[outputs.size()];
for (int i = 0; i < nOutputPartitions.length; ++i) {
- nOutputPartitions[i] = nPartMap.get(jobRun.getJobActivityGraph()
- .getConsumerActivity(outputs.get(i).getConnectorId()).getOperatorDescriptorId());
+ ConnectorDescriptorId cdId = outputs.get(i).getConnectorId();
+ ActivityId aid = ac.getConsumerActivity(cdId);
+ Integer nPartInt = nPartMap.get(aid.getOperatorDescriptorId());
+ nOutputPartitions[i] = nPartInt;
}
}
ActivityPartitionDetails apd = new ActivityPartitionDetails(nParts, nInputPartitions, nOutputPartitions);
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/JobScheduler.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/JobScheduler.java
index e0eee00..b163db5 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/JobScheduler.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/JobScheduler.java
@@ -15,6 +15,7 @@
package edu.uci.ics.hyracks.control.cc.scheduler;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -25,34 +26,25 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
-import edu.uci.ics.hyracks.api.comm.NetworkAddress;
import edu.uci.ics.hyracks.api.constraints.Constraint;
-import edu.uci.ics.hyracks.api.constraints.IConstraintAcceptor;
import edu.uci.ics.hyracks.api.constraints.expressions.LValueConstraintExpression;
import edu.uci.ics.hyracks.api.constraints.expressions.PartitionLocationExpression;
import edu.uci.ics.hyracks.api.dataflow.ActivityId;
import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.dataflow.TaskId;
import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.job.JobActivityGraph;
+import edu.uci.ics.hyracks.api.job.ActivityCluster;
+import edu.uci.ics.hyracks.api.job.ActivityClusterGraph;
import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.api.job.JobStatus;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
import edu.uci.ics.hyracks.api.util.JavaSerializationUtils;
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
import edu.uci.ics.hyracks.control.cc.NodeControllerState;
-import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
-import edu.uci.ics.hyracks.control.cc.job.ActivityPlan;
-import edu.uci.ics.hyracks.control.cc.job.IConnectorDescriptorVisitor;
-import edu.uci.ics.hyracks.control.cc.job.IOperatorDescriptorVisitor;
+import edu.uci.ics.hyracks.control.cc.job.ActivityClusterPlan;
import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.job.PlanUtils;
import edu.uci.ics.hyracks.control.cc.job.Task;
import edu.uci.ics.hyracks.control.cc.job.TaskAttempt;
import edu.uci.ics.hyracks.control.cc.job.TaskCluster;
@@ -75,14 +67,13 @@
private final Set<TaskCluster> inProgressTaskClusters;
- private Set<ActivityCluster> rootActivityClusters;
-
- public JobScheduler(ClusterControllerService ccs, JobRun jobRun) {
+ public JobScheduler(ClusterControllerService ccs, JobRun jobRun, Collection<Constraint> constraints) {
this.ccs = ccs;
this.jobRun = jobRun;
solver = new PartitionConstraintSolver();
partitionProducingTaskClusterMap = new HashMap<PartitionId, TaskCluster>();
inProgressTaskClusters = new HashSet<TaskCluster>();
+ solver.addConstraints(constraints);
}
public JobRun getJobRun() {
@@ -94,41 +85,10 @@
}
public void startJob() throws HyracksException {
- analyze();
startRunnableActivityClusters();
}
- private void analyze() throws HyracksException {
- final JobActivityGraph jag = jobRun.getJobActivityGraph();
- final ICCApplicationContext appCtx = ccs.getApplicationMap().get(jag.getApplicationName());
- JobSpecification spec = jag.getJobSpecification();
- final Set<Constraint> contributedConstraints = new HashSet<Constraint>();
- final IConstraintAcceptor acceptor = new IConstraintAcceptor() {
- @Override
- public void addConstraint(Constraint constraint) {
- contributedConstraints.add(constraint);
- }
- };
- PlanUtils.visit(spec, new IOperatorDescriptorVisitor() {
- @Override
- public void visit(IOperatorDescriptor op) {
- op.contributeSchedulingConstraints(acceptor, jag, appCtx);
- }
- });
- PlanUtils.visit(spec, new IConnectorDescriptorVisitor() {
- @Override
- public void visit(IConnectorDescriptor conn) {
- conn.contributeSchedulingConstraints(acceptor, jag, appCtx);
- }
- });
- contributedConstraints.addAll(spec.getUserConstraints());
- solver.addConstraints(contributedConstraints);
-
- ActivityClusterGraphBuilder acgb = new ActivityClusterGraphBuilder(jobRun);
- rootActivityClusters = acgb.inferActivityClusters(jag);
- }
-
- private void findRunnableTaskClusterRoots(Set<TaskCluster> frontier, Set<ActivityCluster> roots)
+ private void findRunnableTaskClusterRoots(Set<TaskCluster> frontier, Collection<ActivityCluster> roots)
throws HyracksException {
for (ActivityCluster root : roots) {
findRunnableTaskClusterRoots(frontier, root);
@@ -144,14 +104,13 @@
findRunnableTaskClusterRoots(frontier, depAC);
} else {
boolean tcRootsComplete = true;
- Set<TaskCluster> depACTCRoots = new HashSet<TaskCluster>();
- for (TaskCluster tc : depAC.getPlan().getTaskClusters()) {
+ for (TaskCluster tc : getActivityClusterPlan(depAC).getTaskClusters()) {
if (tc.getProducedPartitions().isEmpty()) {
TaskClusterAttempt tca = findLastTaskClusterAttempt(tc);
if (tca == null || tca.getStatus() != TaskClusterAttempt.TaskClusterStatus.COMPLETED) {
tcRootsComplete = false;
+ break;
}
- depACTCRoots.add(tc);
}
}
if (!tcRootsComplete) {
@@ -163,10 +122,11 @@
if (depsComplete) {
if (!isPlanned(candidate)) {
ActivityClusterPlanner acp = new ActivityClusterPlanner(this);
- acp.planActivityCluster(candidate);
+ ActivityClusterPlan acPlan = acp.planActivityCluster(candidate);
+ jobRun.getActivityClusterPlanMap().put(candidate.getId(), acPlan);
partitionProducingTaskClusterMap.putAll(acp.getPartitionProducingTaskClusterMap());
}
- for (TaskCluster tc : candidate.getPlan().getTaskClusters()) {
+ for (TaskCluster tc : getActivityClusterPlan(candidate).getTaskClusters()) {
if (tc.getProducedPartitions().isEmpty()) {
TaskClusterAttempt tca = findLastTaskClusterAttempt(tc);
if (tca == null || tca.getStatus() != TaskClusterAttempt.TaskClusterStatus.COMPLETED) {
@@ -177,13 +137,18 @@
}
}
+ private ActivityClusterPlan getActivityClusterPlan(ActivityCluster ac) {
+ return jobRun.getActivityClusterPlanMap().get(ac.getId());
+ }
+
private boolean isPlanned(ActivityCluster ac) {
- return ac.getPlan() != null;
+ return jobRun.getActivityClusterPlanMap().get(ac.getId()) != null;
}
private void startRunnableActivityClusters() throws HyracksException {
Set<TaskCluster> taskClusterRoots = new HashSet<TaskCluster>();
- findRunnableTaskClusterRoots(taskClusterRoots, rootActivityClusters);
+ findRunnableTaskClusterRoots(taskClusterRoots, jobRun.getActivityClusterGraph().getActivityClusterMap()
+ .values());
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("Runnable TC roots: " + taskClusterRoots + ", inProgressTaskClusters: "
+ inProgressTaskClusters);
@@ -323,7 +288,7 @@
private void assignTaskLocations(TaskCluster tc, Map<String, List<TaskAttemptDescriptor>> taskAttemptMap)
throws HyracksException {
- JobActivityGraph jag = jobRun.getJobActivityGraph();
+ ActivityClusterGraph acg = jobRun.getActivityClusterGraph();
Task[] tasks = tc.getTasks();
List<TaskClusterAttempt> tcAttempts = tc.getAttempts();
int attempts = tcAttempts.size();
@@ -346,7 +311,7 @@
Task ts = tasks[i];
TaskId tid = ts.getTaskId();
TaskAttempt taskAttempt = taskAttempts.get(tid);
- String nodeId = assignLocation(jag, locationMap, tid, taskAttempt);
+ String nodeId = assignLocation(acg, locationMap, tid, taskAttempt);
taskAttempt.setNodeId(nodeId);
taskAttempt.setStatus(TaskAttempt.TaskStatus.RUNNING, null);
taskAttempt.setStartTime(System.currentTimeMillis());
@@ -392,10 +357,11 @@
inProgressTaskClusters.add(tc);
}
- private String assignLocation(JobActivityGraph jag, Map<TaskId, LValueConstraintExpression> locationMap,
+ private String assignLocation(ActivityClusterGraph acg, Map<TaskId, LValueConstraintExpression> locationMap,
TaskId tid, TaskAttempt taskAttempt) throws HyracksException {
ActivityId aid = tid.getActivityId();
- Set<ActivityId> blockers = jag.getBlocked2BlockerMap().get(aid);
+ ActivityCluster ac = acg.getActivityMap().get(aid);
+ Set<ActivityId> blockers = ac.getBlocked2BlockerMap().get(aid);
String nodeId = null;
if (blockers != null) {
for (ActivityId blocker : blockers) {
@@ -441,8 +407,8 @@
private String findTaskLocation(TaskId tid) {
ActivityId aid = tid.getActivityId();
- ActivityCluster ac = jobRun.getActivityClusterMap().get(aid);
- Task[] tasks = ac.getPlan().getActivityPlanMap().get(aid).getTasks();
+ ActivityCluster ac = jobRun.getActivityClusterGraph().getActivityMap().get(aid);
+ Task[] tasks = getActivityClusterPlan(ac).getActivityPlanMap().get(aid).getTasks();
List<TaskClusterAttempt> tcAttempts = tasks[tid.getPartition()].getTaskCluster().getAttempts();
if (tcAttempts == null || tcAttempts.isEmpty()) {
return null;
@@ -462,9 +428,10 @@
private void startTasks(Map<String, List<TaskAttemptDescriptor>> taskAttemptMap) throws HyracksException {
final JobId jobId = jobRun.getJobId();
- final JobActivityGraph jag = jobRun.getJobActivityGraph();
- final String appName = jag.getApplicationName();
- final Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies = jobRun.getConnectorPolicyMap();
+ final ActivityClusterGraph acg = jobRun.getActivityClusterGraph();
+ final String appName = jobRun.getApplicationName();
+ final Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies = new HashMap<ConnectorDescriptorId, IConnectorPolicy>(
+ jobRun.getConnectorPolicyMap());
for (Map.Entry<String, List<TaskAttemptDescriptor>> entry : taskAttemptMap.entrySet()) {
String nodeId = entry.getKey();
final List<TaskAttemptDescriptor> taskDescriptors = entry.getValue();
@@ -476,8 +443,9 @@
LOGGER.fine("Starting: " + taskDescriptors + " at " + entry.getKey());
}
try {
- byte[] jagBytes = changed ? JavaSerializationUtils.serialize(jag) : null;
- node.getNodeController().startTasks(appName, jobId, jagBytes, taskDescriptors, connectorPolicies);
+ byte[] jagBytes = changed ? JavaSerializationUtils.serialize(acg) : null;
+ node.getNodeController().startTasks(appName, jobId, jagBytes, taskDescriptors, connectorPolicies,
+ jobRun.getFlags());
} catch (Exception e) {
e.printStackTrace();
}
@@ -642,7 +610,7 @@
lastAttempt.setStatus(TaskClusterAttempt.TaskClusterStatus.FAILED);
lastAttempt.setEndTime(System.currentTimeMillis());
abortDoomedTaskClusters();
- if (lastAttempt.getAttempt() >= ac.getMaxTaskClusterReattempts()) {
+ if (lastAttempt.getAttempt() >= jobRun.getActivityClusterGraph().getMaxReattempts()) {
abortJob(new HyracksException(details));
return;
}
@@ -665,8 +633,8 @@
public void notifyNodeFailures(Set<String> deadNodes) {
try {
jobRun.getPartitionMatchMaker().notifyNodeFailures(deadNodes);
- for (ActivityCluster ac : jobRun.getActivityClusters()) {
- TaskCluster[] taskClusters = ac.getPlan().getTaskClusters();
+ for (ActivityCluster ac : jobRun.getActivityClusterGraph().getActivityClusterMap().values()) {
+ TaskCluster[] taskClusters = getActivityClusterPlan(ac).getTaskClusters();
if (taskClusters != null) {
for (TaskCluster tc : taskClusters) {
TaskClusterAttempt lastTaskClusterAttempt = findLastTaskClusterAttempt(tc);
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/JobsRESTAPIFunction.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/JobsRESTAPIFunction.java
index 4135b18..98cef8d 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/JobsRESTAPIFunction.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/JobsRESTAPIFunction.java
@@ -19,9 +19,8 @@
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
import edu.uci.ics.hyracks.control.cc.web.util.IJSONOutputFunction;
-import edu.uci.ics.hyracks.control.cc.work.GetJobActivityGraphJSONWork;
+import edu.uci.ics.hyracks.control.cc.work.GetActivityClusterGraphJSONWork;
import edu.uci.ics.hyracks.control.cc.work.GetJobRunJSONWork;
-import edu.uci.ics.hyracks.control.cc.work.GetJobSpecificationJSONWork;
import edu.uci.ics.hyracks.control.cc.work.GetJobSummariesJSONWork;
public class JobsRESTAPIFunction implements IJSONOutputFunction {
@@ -49,12 +48,8 @@
case 2: {
JobId jobId = JobId.parse(arguments[0]);
- if ("job-specification".equalsIgnoreCase(arguments[1])) {
- GetJobSpecificationJSONWork gjse = new GetJobSpecificationJSONWork(ccs, jobId);
- ccs.getWorkQueue().scheduleAndSync(gjse);
- result.put("result", gjse.getJSON());
- } else if ("job-activity-graph".equalsIgnoreCase(arguments[1])) {
- GetJobActivityGraphJSONWork gjage = new GetJobActivityGraphJSONWork(ccs, jobId);
+ if ("job-activity-graph".equalsIgnoreCase(arguments[1])) {
+ GetActivityClusterGraphJSONWork gjage = new GetActivityClusterGraphJSONWork(ccs, jobId);
ccs.getWorkQueue().scheduleAndSync(gjage);
result.put("result", gjage.getJSON());
} else if ("job-run".equalsIgnoreCase(arguments[1])) {
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/AbstractTaskLifecycleWork.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/AbstractTaskLifecycleWork.java
index 01f14f3..8a552f5 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/AbstractTaskLifecycleWork.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/AbstractTaskLifecycleWork.java
@@ -20,9 +20,9 @@
import edu.uci.ics.hyracks.api.dataflow.ActivityId;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.dataflow.TaskId;
+import edu.uci.ics.hyracks.api.job.ActivityCluster;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
import edu.uci.ics.hyracks.control.cc.job.ActivityPlan;
import edu.uci.ics.hyracks.control.cc.job.JobRun;
import edu.uci.ics.hyracks.control.cc.job.Task;
@@ -49,10 +49,11 @@
JobRun run = ccs.getActiveRunMap().get(jobId);
if (run != null) {
TaskId tid = taId.getTaskId();
- Map<ActivityId, ActivityCluster> activityClusterMap = run.getActivityClusterMap();
+ Map<ActivityId, ActivityCluster> activityClusterMap = run.getActivityClusterGraph().getActivityMap();
ActivityCluster ac = activityClusterMap.get(tid.getActivityId());
if (ac != null) {
- Map<ActivityId, ActivityPlan> taskStateMap = ac.getPlan().getActivityPlanMap();
+ Map<ActivityId, ActivityPlan> taskStateMap = run.getActivityClusterPlanMap().get(ac.getId())
+ .getActivityPlanMap();
Task[] taskStates = taskStateMap.get(tid.getActivityId()).getTasks();
if (taskStates != null && taskStates.length > tid.getPartition()) {
Task ts = taskStates[tid.getPartition()];
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationMessageWork.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationMessageWork.java
new file mode 100644
index 0000000..22ff84d
--- /dev/null
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationMessageWork.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.messages.IMessage;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.common.application.ApplicationContext;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+
+/**
+ * @author rico
+ *
+ */
+public class ApplicationMessageWork extends AbstractWork {
+
+ private static final Logger LOGGER = Logger.getLogger(ApplicationMessageWork.class.getName());
+ private byte[] message;
+ private String nodeId;
+ private ClusterControllerService ccs;
+ private String appName;
+
+ public ApplicationMessageWork(ClusterControllerService ccs, byte[] message, String appName, String nodeId) {
+ this.ccs = ccs;
+ this.nodeId = nodeId;
+ this.message = message;
+ this.appName = appName;
+ }
+
+ @Override
+ public void run() {
+
+ final ApplicationContext ctx = ccs.getApplicationMap().get(appName);
+ try {
+ final IMessage data = (IMessage) ctx.deserialize(message);
+ (new Thread() {
+ public void run() {
+ ctx.getMessageBroker().receivedMessage(data, nodeId);
+ }
+ }).start();
+ } catch (IOException e) {
+ LOGGER.log(Level.WARNING, "Error in stats reporting", e);
+ } catch (ClassNotFoundException e) {
+ Logger.getLogger(this.getClass().getName()).log(Level.WARNING, "Error in stats reporting", e);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "nodeID: " + nodeId;
+ }
+
+}
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobActivityGraphJSONWork.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetActivityClusterGraphJSONWork.java
similarity index 87%
rename from hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobActivityGraphJSONWork.java
rename to hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetActivityClusterGraphJSONWork.java
index 677bece..1f79763 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobActivityGraphJSONWork.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetActivityClusterGraphJSONWork.java
@@ -21,12 +21,12 @@
import edu.uci.ics.hyracks.control.cc.job.JobRun;
import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
-public class GetJobActivityGraphJSONWork extends SynchronizableWork {
+public class GetActivityClusterGraphJSONWork extends SynchronizableWork {
private final ClusterControllerService ccs;
private final JobId jobId;
private JSONObject json;
- public GetJobActivityGraphJSONWork(ClusterControllerService ccs, JobId jobId) {
+ public GetActivityClusterGraphJSONWork(ClusterControllerService ccs, JobId jobId) {
this.ccs = ccs;
this.jobId = jobId;
}
@@ -41,7 +41,7 @@
return;
}
}
- json = run.getJobActivityGraph().toJSON();
+ json = run.getActivityClusterGraph().toJSON();
}
public JSONObject getJSON() {
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobSpecificationJSONWork.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobSpecificationJSONWork.java
deleted file mode 100644
index 05b7cae..0000000
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobSpecificationJSONWork.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.work;
-
-import org.json.JSONObject;
-
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
-
-public class GetJobSpecificationJSONWork extends SynchronizableWork {
- private final ClusterControllerService ccs;
- private final JobId jobId;
- private JSONObject json;
-
- public GetJobSpecificationJSONWork(ClusterControllerService ccs, JobId jobId) {
- this.ccs = ccs;
- this.jobId = jobId;
- }
-
- @Override
- protected void doRun() throws Exception {
- JobRun run = ccs.getActiveRunMap().get(jobId);
- if (run == null) {
- run = ccs.getRunMapArchive().get(jobId);
- if (run == null) {
- json = new JSONObject();
- return;
- }
- }
- json = run.getJobActivityGraph().getJobSpecification().toJSON();
- }
-
- public JSONObject getJSON() {
- return json;
- }
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobSummariesJSONWork.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobSummariesJSONWork.java
index 5cc31f6..a0afd61 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobSummariesJSONWork.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobSummariesJSONWork.java
@@ -44,7 +44,7 @@
JSONObject jo = new JSONObject();
jo.put("type", "job-summary");
jo.put("job-id", run.getJobId().toString());
- jo.put("application-name", run.getJobActivityGraph().getApplicationName());
+ jo.put("application-name", run.getApplicationName());
jo.put("create-time", run.getCreateTime());
jo.put("start-time", run.getCreateTime());
jo.put("end-time", run.getCreateTime());
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCleanupWork.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCleanupWork.java
index 07088bb..6b5ff03 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCleanupWork.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCleanupWork.java
@@ -17,10 +17,16 @@
import java.util.Set;
import java.util.logging.Logger;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.job.ActivityClusterGraph;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobStatus;
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
import edu.uci.ics.hyracks.control.cc.NodeControllerState;
+import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
import edu.uci.ics.hyracks.control.cc.job.JobRun;
import edu.uci.ics.hyracks.control.common.work.AbstractWork;
@@ -53,13 +59,44 @@
Set<String> targetNodes = run.getParticipatingNodeIds();
run.getCleanupPendingNodeIds().addAll(targetNodes);
run.setPendingStatus(status, exception);
- for (String n : targetNodes) {
- NodeControllerState ncs = ccs.getNodeMap().get(n);
+ if (targetNodes != null && !targetNodes.isEmpty()) {
+ for (String n : targetNodes) {
+ NodeControllerState ncs = ccs.getNodeMap().get(n);
+ try {
+ ncs.getNodeController().cleanUpJoblet(jobId, status);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ } else {
+ CCApplicationContext appCtx = ccs.getApplicationMap().get(run.getApplicationName());
+ if (appCtx != null) {
+ try {
+ appCtx.notifyJobFinish(jobId);
+ } catch (HyracksException e) {
+ e.printStackTrace();
+ }
+ }
+ run.setStatus(run.getPendingStatus(), run.getPendingException());
+ ccs.getActiveRunMap().remove(jobId);
+ ccs.getRunMapArchive().put(jobId, run);
try {
- ncs.getNodeController().cleanUpJoblet(jobId, status);
+ ccs.getJobLogFile().log(createJobLogObject(run));
} catch (Exception e) {
- e.printStackTrace();
+ throw new RuntimeException(e);
}
}
}
+
+ private JSONObject createJobLogObject(final JobRun run) {
+ JSONObject jobLogObject = new JSONObject();
+ try {
+ ActivityClusterGraph acg = run.getActivityClusterGraph();
+ jobLogObject.put("activity-cluster-graph", acg.toJSON());
+ jobLogObject.put("job-run", run.toJSON());
+ } catch (JSONException e) {
+ throw new RuntimeException(e);
+ }
+ return jobLogObject;
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCreateWork.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCreateWork.java
deleted file mode 100644
index b7cf629..0000000
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCreateWork.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.work;
-
-import java.util.EnumSet;
-
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.job.JobActivityGraph;
-import edu.uci.ics.hyracks.api.job.JobFlag;
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.api.job.JobStatus;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
-import edu.uci.ics.hyracks.control.cc.job.IOperatorDescriptorVisitor;
-import edu.uci.ics.hyracks.control.cc.job.JobActivityGraphBuilder;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.job.PlanUtils;
-import edu.uci.ics.hyracks.control.cc.scheduler.JobScheduler;
-import edu.uci.ics.hyracks.control.common.work.IResultCallback;
-import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
-
-public class JobCreateWork extends SynchronizableWork {
- private final ClusterControllerService ccs;
- private final byte[] jobSpec;
- private final EnumSet<JobFlag> jobFlags;
- private final JobId jobId;
- private final String appName;
- private final IResultCallback<JobId> callback;
-
- public JobCreateWork(ClusterControllerService ccs, JobId jobId, String appName, byte[] jobSpec,
- EnumSet<JobFlag> jobFlags, IResultCallback<JobId> callback) {
- this.jobId = jobId;
- this.ccs = ccs;
- this.jobSpec = jobSpec;
- this.jobFlags = jobFlags;
- this.appName = appName;
- this.callback = callback;
- }
-
- @Override
- protected void doRun() throws Exception {
- try {
- CCApplicationContext appCtx = ccs.getApplicationMap().get(appName);
- if (appCtx == null) {
- throw new HyracksException("No application with id " + appName + " found");
- }
- JobSpecification spec = appCtx.createJobSpecification(jobSpec);
-
- final JobActivityGraphBuilder builder = new JobActivityGraphBuilder();
- builder.init(appName, spec, jobFlags);
- PlanUtils.visit(spec, new IOperatorDescriptorVisitor() {
- @Override
- public void visit(IOperatorDescriptor op) {
- op.contributeActivities(builder);
- }
- });
- final JobActivityGraph jag = builder.getActivityGraph();
-
- JobRun run = new JobRun(jobId, jag);
-
- run.setStatus(JobStatus.INITIALIZED, null);
-
- ccs.getActiveRunMap().put(jobId, run);
- JobScheduler jrs = new JobScheduler(ccs, run);
- run.setScheduler(jrs);
- appCtx.notifyJobCreation(jobId, spec);
- callback.setValue(jobId);
- } catch (Exception e) {
- callback.setException(e);
- return;
- }
- }
-
- public JobId getJobId() {
- return jobId;
- }
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobStartWork.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobStartWork.java
index dd834ed..b6a33cd 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobStartWork.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobStartWork.java
@@ -14,41 +14,59 @@
*/
package edu.uci.ics.hyracks.control.cc.work;
+import java.util.EnumSet;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.job.IActivityClusterGraphGenerator;
+import edu.uci.ics.hyracks.api.job.IActivityClusterGraphGeneratorFactory;
+import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobStatus;
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
import edu.uci.ics.hyracks.control.cc.job.JobRun;
import edu.uci.ics.hyracks.control.common.work.IResultCallback;
import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
public class JobStartWork extends SynchronizableWork {
private final ClusterControllerService ccs;
+ private final byte[] acggfBytes;
+ private final EnumSet<JobFlag> jobFlags;
private final JobId jobId;
- private final IResultCallback<Object> callback;
+ private final String appName;
+ private final IResultCallback<JobId> callback;
- public JobStartWork(ClusterControllerService ccs, JobId jobId, IResultCallback<Object> callback) {
- this.ccs = ccs;
+ public JobStartWork(ClusterControllerService ccs, String appName, byte[] acggfBytes, EnumSet<JobFlag> jobFlags,
+ JobId jobId, IResultCallback<JobId> callback) {
this.jobId = jobId;
+ this.ccs = ccs;
+ this.acggfBytes = acggfBytes;
+ this.jobFlags = jobFlags;
+ this.appName = appName;
this.callback = callback;
}
@Override
protected void doRun() throws Exception {
try {
- JobRun run = ccs.getActiveRunMap().get(jobId);
- if (run == null) {
- throw new Exception("Unable to find job with id = " + jobId);
+ final CCApplicationContext appCtx = ccs.getApplicationMap().get(appName);
+ if (appCtx == null) {
+ throw new HyracksException("No application with id " + appName + " found");
}
- if (run.getStatus() != JobStatus.INITIALIZED) {
- throw new Exception("Job already started");
- }
+ IActivityClusterGraphGeneratorFactory acggf = appCtx.createActivityClusterGraphGeneratorFactory(acggfBytes);
+ IActivityClusterGraphGenerator acgg = acggf.createActivityClusterGraphGenerator(appName, jobId, appCtx,
+ jobFlags);
+ JobRun run = new JobRun(ccs, jobId, appName, acgg, jobFlags);
+ run.setStatus(JobStatus.INITIALIZED, null);
+ ccs.getActiveRunMap().put(jobId, run);
+ appCtx.notifyJobCreation(jobId, acggf);
run.setStatus(JobStatus.RUNNING, null);
try {
run.getScheduler().startJob();
} catch (Exception e) {
ccs.getWorkQueue().schedule(new JobCleanupWork(ccs, run.getJobId(), JobStatus.FAILURE, e));
}
- callback.setValue(null);
+ callback.setValue(jobId);
} catch (Exception e) {
callback.setException(e);
}
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobletCleanupNotificationWork.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobletCleanupNotificationWork.java
index 80ab5ec..9927289 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobletCleanupNotificationWork.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobletCleanupNotificationWork.java
@@ -22,7 +22,7 @@
import org.json.JSONObject;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.job.JobActivityGraph;
+import edu.uci.ics.hyracks.api.job.ActivityClusterGraph;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
import edu.uci.ics.hyracks.control.cc.NodeControllerState;
@@ -59,7 +59,7 @@
ncs.getActiveJobIds().remove(jobId);
}
if (cleanupPendingNodes.isEmpty()) {
- CCApplicationContext appCtx = ccs.getApplicationMap().get(run.getJobActivityGraph().getApplicationName());
+ CCApplicationContext appCtx = ccs.getApplicationMap().get(run.getApplicationName());
if (appCtx != null) {
try {
appCtx.notifyJobFinish(jobId);
@@ -82,9 +82,8 @@
private JSONObject createJobLogObject(final JobRun run) {
JSONObject jobLogObject = new JSONObject();
try {
- JobActivityGraph jag = run.getJobActivityGraph();
- jobLogObject.put("job-specification", jag.getJobSpecification().toJSON());
- jobLogObject.put("job-activity-graph", jag.toJSON());
+ ActivityClusterGraph acg = run.getActivityClusterGraph();
+ jobLogObject.put("activity-cluster-graph", acg.toJSON());
jobLogObject.put("job-run", run.toJSON());
} catch (JSONException e) {
throw new RuntimeException(e);
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionAvailibilityWork.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionAvailibilityWork.java
index 2456c25..2a844ad 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionAvailibilityWork.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionAvailibilityWork.java
@@ -15,7 +15,6 @@
package edu.uci.ics.hyracks.control.cc.work;
import java.util.List;
-import java.util.logging.Level;
import org.apache.commons.lang3.tuple.Pair;
@@ -57,11 +56,6 @@
}
@Override
- public Level logLevel() {
- return Level.FINEST;
- }
-
- @Override
public String toString() {
return "PartitionAvailable@" + partitionDescriptor;
}
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskCompleteWork.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskCompleteWork.java
index 2694af4..fee0832 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskCompleteWork.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskCompleteWork.java
@@ -20,7 +20,6 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
import edu.uci.ics.hyracks.control.cc.job.JobRun;
import edu.uci.ics.hyracks.control.cc.job.TaskAttempt;
import edu.uci.ics.hyracks.control.common.job.profiling.om.JobProfile;
@@ -39,8 +38,7 @@
@Override
protected void performEvent(TaskAttempt ta) {
try {
- ActivityCluster ac = ta.getTask().getTaskCluster().getActivityCluster();
- JobRun run = ac.getJobRun();
+ JobRun run = ccs.getActiveRunMap().get(jobId);
if (statistics != null) {
JobProfile jobProfile = run.getJobProfile();
Map<String, JobletProfile> jobletProfiles = jobProfile.getJobletProfiles();
@@ -51,7 +49,7 @@
}
jobletProfile.getTaskProfiles().put(taId, statistics);
}
- ac.getJobRun().getScheduler().notifyTaskComplete(ta);
+ run.getScheduler().notifyTaskComplete(ta);
} catch (HyracksException e) {
e.printStackTrace();
}
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskFailureWork.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskFailureWork.java
index 3201bec..0fff257 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskFailureWork.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskFailureWork.java
@@ -15,24 +15,25 @@
package edu.uci.ics.hyracks.control.cc.work;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.job.ActivityCluster;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
+import edu.uci.ics.hyracks.control.cc.job.JobRun;
import edu.uci.ics.hyracks.control.cc.job.TaskAttempt;
public class TaskFailureWork extends AbstractTaskLifecycleWork {
private final String details;
- public TaskFailureWork(ClusterControllerService ccs, JobId jobId, TaskAttemptId taId, String nodeId,
- String details) {
+ public TaskFailureWork(ClusterControllerService ccs, JobId jobId, TaskAttemptId taId, String nodeId, String details) {
super(ccs, jobId, taId, nodeId);
this.details = details;
}
@Override
protected void performEvent(TaskAttempt ta) {
+ JobRun run = ccs.getActiveRunMap().get(jobId);
ActivityCluster ac = ta.getTask().getTaskCluster().getActivityCluster();
- ac.getJobRun().getScheduler().notifyTaskFailure(ta, ac, details);
+ run.getScheduler().notifyTaskFailure(ta, ac, details);
}
@Override
diff --git a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationContext.java b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationContext.java
index 9f29fbd..fc755a5 100644
--- a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationContext.java
+++ b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationContext.java
@@ -38,6 +38,7 @@
import edu.uci.ics.hyracks.api.application.IApplicationContext;
import edu.uci.ics.hyracks.api.application.IBootstrap;
+import edu.uci.ics.hyracks.api.messages.IMessageBroker;
import edu.uci.ics.hyracks.api.util.JavaSerializationUtils;
import edu.uci.ics.hyracks.control.common.context.ServerContext;
@@ -54,6 +55,7 @@
protected Properties deploymentDescriptor;
protected IBootstrap bootstrap;
protected Serializable distributedState;
+ protected IMessageBroker messageBroker;
public ApplicationContext(ServerContext serverCtx, String appName) throws IOException {
this.serverCtx = serverCtx;
@@ -205,4 +207,14 @@
public ApplicationStatus getStatus() {
return status;
}
+
+ @Override
+ public void setMessageBroker(IMessageBroker messageBroker) {
+ this.messageBroker = messageBroker;
+ }
+
+ @Override
+ public IMessageBroker getMessageBroker() {
+ return this.messageBroker;
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/IClusterController.java b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/IClusterController.java
index a03f5c9..a25250a 100644
--- a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/IClusterController.java
+++ b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/IClusterController.java
@@ -18,6 +18,7 @@
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.messages.IMessage;
import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
import edu.uci.ics.hyracks.control.common.controllers.NodeRegistration;
import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatData;
@@ -47,4 +48,6 @@
public void registerPartitionRequest(PartitionRequest partitionRequest) throws Exception;
public void notifyApplicationStateChange(String nodeId, String appName, ApplicationStatus status) throws Exception;
+
+ public void sendApplicationMessageToCC(byte[] data, String appName, String nodeId) throws Exception;
}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/INodeController.java b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/INodeController.java
index 47f4ceb..049adf8 100644
--- a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/INodeController.java
+++ b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/INodeController.java
@@ -14,6 +14,7 @@
*/
package edu.uci.ics.hyracks.control.common.base;
+import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -21,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
+import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobStatus;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
@@ -28,7 +30,7 @@
public interface INodeController {
public void startTasks(String appName, JobId jobId, byte[] planBytes, List<TaskAttemptDescriptor> taskDescriptors,
- Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies) throws Exception;
+ Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies, EnumSet<JobFlag> flags) throws Exception;
public void abortTasks(JobId jobId, List<TaskAttemptId> tasks) throws Exception;
diff --git a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/CCConfig.java b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/CCConfig.java
index 350f527..6c208fe 100644
--- a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/CCConfig.java
+++ b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/CCConfig.java
@@ -14,6 +14,7 @@
*/
package edu.uci.ics.hyracks.control.common.controllers;
+import java.io.File;
import java.util.List;
import org.kohsuke.args4j.Option;
@@ -52,6 +53,9 @@
@Option(name = "-cc-root", usage = "Sets the root folder used for file operations. (default: ClusterControllerService)")
public String ccRoot = "ClusterControllerService";
+ @Option(name = "-cluster-topology", usage = "Sets the XML file that defines the cluster topology. (default: null)")
+ public File clusterTopologyDefinition = null;
+
public void toCommandLine(List<String> cList) {
cList.add("-client-net-ip-address");
cList.add(clientNetIpAddress);
@@ -75,5 +79,9 @@
cList.add(String.valueOf(jobHistorySize));
cList.add("-cc-root");
cList.add(ccRoot);
+ if (clusterTopologyDefinition != null) {
+ cList.add("-cluster-topology");
+ cList.add(clusterTopologyDefinition.getAbsolutePath());
+ }
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NCConfig.java b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NCConfig.java
index 574f552..167eb4b 100644
--- a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NCConfig.java
+++ b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NCConfig.java
@@ -37,9 +37,6 @@
@Option(name = "-data-ip-address", usage = "IP Address to bind data listener", required = true)
public String dataIPAddress;
- @Option(name = "-frame-size", usage = "Frame Size to use for data communication (default: 32768)")
- public int frameSize = 32768;
-
@Option(name = "-iodevices", usage = "Comma separated list of IO Device mount points (default: One device in default temp folder)", required = false)
public String ioDevices = System.getProperty("java.io.tmpdir");
@@ -55,6 +52,9 @@
@Option(name = "-net-thread-count", usage = "Number of threads to use for Network I/O (default: 1)")
public int nNetThreads = 1;
+ @Option(name = "-max-memory", usage = "Maximum memory usable at this Node Controller in bytes (default: -1 auto)")
+ public int maxMemory = -1;
+
public void toCommandLine(List<String> cList) {
cList.add("-cc-host");
cList.add(ccHost);
@@ -66,8 +66,6 @@
cList.add(nodeId);
cList.add("-data-ip-address");
cList.add(dataIPAddress);
- cList.add("-frame-size");
- cList.add(String.valueOf(frameSize));
cList.add("-iodevices");
cList.add(ioDevices);
cList.add("-dcache-client-servers");
@@ -80,5 +78,7 @@
cList.add(dcacheClientPath);
cList.add("-net-thread-count");
cList.add(String.valueOf(nNetThreads));
+ cList.add("-max-memory");
+ cList.add(String.valueOf(maxMemory));
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/CCNCFunctions.java b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/CCNCFunctions.java
index 9ea4636..8f0056f 100644
--- a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/CCNCFunctions.java
+++ b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/CCNCFunctions.java
@@ -22,8 +22,11 @@
import java.io.OutputStream;
import java.io.Serializable;
import java.nio.ByteBuffer;
+import java.util.EnumSet;
import java.util.List;
import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
import edu.uci.ics.hyracks.api.comm.NetworkAddress;
import edu.uci.ics.hyracks.api.dataflow.ActivityId;
@@ -32,6 +35,7 @@
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.dataflow.TaskId;
import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
+import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobStatus;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
@@ -49,6 +53,8 @@
import edu.uci.ics.hyracks.ipc.impl.JavaSerializationBasedPayloadSerializerDeserializer;
public class CCNCFunctions {
+ private static final Logger LOGGER = Logger.getLogger(CCNCFunctions.class.getName());
+
private static final int FID_CODE_SIZE = 1;
public enum FunctionId {
@@ -70,10 +76,47 @@
CREATE_APPLICATION,
DESTROY_APPLICATION,
REPORT_PARTITION_AVAILABILITY,
+ SEND_APPLICATION_MESSAGE,
OTHER
}
+ public static class SendApplicationMessageFunction extends Function {
+ private static final long serialVersionUID = 1L;
+ private byte[] serializedMessage;
+ private String nodeId;
+ private String appName;
+
+ public String getNodeId() {
+ return nodeId;
+ }
+
+ public void setNodeId(String nodeId) {
+ this.nodeId = nodeId;
+ }
+
+ public byte[] getMessage() {
+ return serializedMessage;
+ }
+
+ public SendApplicationMessageFunction(byte[] data, String appName, String nodeId) {
+ super();
+ this.serializedMessage = data;
+ this.nodeId = nodeId;
+ this.appName = appName;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.SEND_APPLICATION_MESSAGE;
+ }
+
+ public String getAppName() {
+ return appName;
+ }
+
+ }
+
public static abstract class Function implements Serializable {
private static final long serialVersionUID = 1L;
@@ -457,15 +500,17 @@
private final byte[] planBytes;
private final List<TaskAttemptDescriptor> taskDescriptors;
private final Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies;
+ private final EnumSet<JobFlag> flags;
public StartTasksFunction(String appName, JobId jobId, byte[] planBytes,
List<TaskAttemptDescriptor> taskDescriptors,
- Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies) {
+ Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies, EnumSet<JobFlag> flags) {
this.appName = appName;
this.jobId = jobId;
this.planBytes = planBytes;
this.taskDescriptors = taskDescriptors;
this.connectorPolicies = connectorPolicies;
+ this.flags = flags;
}
@Override
@@ -492,6 +537,10 @@
public Map<ConnectorDescriptorId, IConnectorPolicy> getConnectorPolicies() {
return connectorPolicies;
}
+
+ public EnumSet<JobFlag> getFlags() {
+ return flags;
+ }
}
public static class AbortTasksFunction extends Function {
@@ -690,8 +739,12 @@
private byte[] serialize(Object object, byte fid) throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
baos.write(fid);
- serialize(baos, object, fid);
- JavaSerializationBasedPayloadSerializerDeserializer.serialize(baos, object);
+ try {
+ serialize(baos, object, fid);
+ } catch (Exception e) {
+ LOGGER.log(Level.SEVERE, "Error serializing " + object, e);
+ throw e;
+ }
baos.close();
return baos.toByteArray();
}
diff --git a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java
index a0dabdd..d789768 100644
--- a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java
+++ b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java
@@ -43,44 +43,40 @@
@Override
public void unregisterNode(String nodeId) throws Exception {
- CCNCFunctions.UnregisterNodeFunction fn = new CCNCFunctions.UnregisterNodeFunction(
- nodeId);
+ CCNCFunctions.UnregisterNodeFunction fn = new CCNCFunctions.UnregisterNodeFunction(nodeId);
ipcHandle.send(-1, fn, null);
}
@Override
public void notifyTaskComplete(JobId jobId, TaskAttemptId taskId, String nodeId, TaskProfile statistics)
throws Exception {
- CCNCFunctions.NotifyTaskCompleteFunction fn = new CCNCFunctions.NotifyTaskCompleteFunction(
- jobId, taskId, nodeId, statistics);
+ CCNCFunctions.NotifyTaskCompleteFunction fn = new CCNCFunctions.NotifyTaskCompleteFunction(jobId, taskId,
+ nodeId, statistics);
ipcHandle.send(-1, fn, null);
}
@Override
public void notifyTaskFailure(JobId jobId, TaskAttemptId taskId, String nodeId, String details) throws Exception {
- CCNCFunctions.NotifyTaskFailureFunction fn = new CCNCFunctions.NotifyTaskFailureFunction(
- jobId, taskId, nodeId, details);
+ CCNCFunctions.NotifyTaskFailureFunction fn = new CCNCFunctions.NotifyTaskFailureFunction(jobId, taskId, nodeId,
+ details);
ipcHandle.send(-1, fn, null);
}
@Override
public void notifyJobletCleanup(JobId jobId, String nodeId) throws Exception {
- CCNCFunctions.NotifyJobletCleanupFunction fn = new CCNCFunctions.NotifyJobletCleanupFunction(
- jobId, nodeId);
+ CCNCFunctions.NotifyJobletCleanupFunction fn = new CCNCFunctions.NotifyJobletCleanupFunction(jobId, nodeId);
ipcHandle.send(-1, fn, null);
}
@Override
public void nodeHeartbeat(String id, HeartbeatData hbData) throws Exception {
- CCNCFunctions.NodeHeartbeatFunction fn = new CCNCFunctions.NodeHeartbeatFunction(id,
- hbData);
+ CCNCFunctions.NodeHeartbeatFunction fn = new CCNCFunctions.NodeHeartbeatFunction(id, hbData);
ipcHandle.send(-1, fn, null);
}
@Override
public void reportProfile(String id, List<JobProfile> profiles) throws Exception {
- CCNCFunctions.ReportProfileFunction fn = new CCNCFunctions.ReportProfileFunction(id,
- profiles);
+ CCNCFunctions.ReportProfileFunction fn = new CCNCFunctions.ReportProfileFunction(id, profiles);
ipcHandle.send(-1, fn, null);
}
@@ -104,4 +100,10 @@
nodeId, appName, status);
ipcHandle.send(-1, fn, null);
}
+
+ @Override
+ public void sendApplicationMessageToCC(byte[] data, String appName, String nodeId) throws Exception {
+ CCNCFunctions.SendApplicationMessageFunction fn = new CCNCFunctions.SendApplicationMessageFunction(data, appName, nodeId);
+ ipcHandle.send(-1, fn, null);
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/NodeControllerRemoteProxy.java b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/NodeControllerRemoteProxy.java
index 8176cb7..10c0a7c 100644
--- a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/NodeControllerRemoteProxy.java
+++ b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/NodeControllerRemoteProxy.java
@@ -14,6 +14,7 @@
*/
package edu.uci.ics.hyracks.control.common.ipc;
+import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -21,6 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
+import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobStatus;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
@@ -37,9 +39,9 @@
@Override
public void startTasks(String appName, JobId jobId, byte[] planBytes, List<TaskAttemptDescriptor> taskDescriptors,
- Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies) throws Exception {
+ Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies, EnumSet<JobFlag> flags) throws Exception {
CCNCFunctions.StartTasksFunction stf = new CCNCFunctions.StartTasksFunction(appName, jobId, planBytes,
- taskDescriptors, connectorPolicies);
+ taskDescriptors, connectorPolicies, flags);
ipcHandle.send(-1, stf, null);
}
diff --git a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/TaskProfile.java b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/TaskProfile.java
index 2116f61..0f82844 100644
--- a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/TaskProfile.java
+++ b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/TaskProfile.java
@@ -14,6 +14,7 @@
*/
package edu.uci.ics.hyracks.control.common.job.profiling.om;
+import java.util.HashMap;
import java.util.Map;
import org.json.JSONArray;
@@ -33,7 +34,7 @@
public TaskProfile(TaskAttemptId taskAttemptId, Map<PartitionId, PartitionProfile> partitionSendProfile) {
this.taskAttemptId = taskAttemptId;
- this.partitionSendProfile = partitionSendProfile;
+ this.partitionSendProfile = new HashMap<PartitionId, PartitionProfile>(partitionSendProfile);
}
public TaskAttemptId getTaskId() {
diff --git a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/WorkQueue.java b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/WorkQueue.java
index 36f0c49..26e5412 100644
--- a/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/WorkQueue.java
+++ b/hyracks/hyracks-control/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/WorkQueue.java
@@ -70,11 +70,11 @@
public void schedule(AbstractWork event) {
enqueueCount.incrementAndGet();
- if (LOGGER.isLoggable(Level.FINE)) {
- LOGGER.fine("Enqueue: " + enqueueCount);
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest("Enqueue: " + enqueueCount);
}
- if (LOGGER.isLoggable(event.logLevel())) {
- LOGGER.log(event.logLevel(), "Scheduling: " + event);
+ if (LOGGER.isLoggable(Level.FINER)) {
+ LOGGER.finer("Scheduling: " + event);
}
queue.offer(event);
}
@@ -105,8 +105,8 @@
continue;
}
dequeueCount.incrementAndGet();
- if (LOGGER.isLoggable(Level.FINE)) {
- LOGGER.fine("Dequeue: " + dequeueCount + "/" + enqueueCount);
+ if (LOGGER.isLoggable(Level.FINEST)) {
+ LOGGER.finest("Dequeue: " + dequeueCount + "/" + enqueueCount);
}
try {
if (LOGGER.isLoggable(r.logLevel())) {
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Joblet.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Joblet.java
index 8f7bc4a..3855b4d 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Joblet.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Joblet.java
@@ -26,16 +26,17 @@
import edu.uci.ics.hyracks.api.comm.PartitionChannel;
import edu.uci.ics.hyracks.api.context.IHyracksJobletContext;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
-import edu.uci.ics.hyracks.api.dataflow.TaskId;
-import edu.uci.ics.hyracks.api.dataflow.state.ITaskState;
+import edu.uci.ics.hyracks.api.dataflow.state.IStateObject;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.api.io.IWorkspaceFileFactory;
+import edu.uci.ics.hyracks.api.job.ActivityClusterGraph;
+import edu.uci.ics.hyracks.api.job.IGlobalJobDataFactory;
import edu.uci.ics.hyracks.api.job.IJobletEventListener;
+import edu.uci.ics.hyracks.api.job.IJobletEventListenerFactory;
import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
-import edu.uci.ics.hyracks.api.job.JobActivityGraph;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobStatus;
import edu.uci.ics.hyracks.api.job.profiling.counters.ICounter;
@@ -59,13 +60,13 @@
private final JobId jobId;
- private final JobActivityGraph jag;
+ private final ActivityClusterGraph acg;
private final Map<PartitionId, IPartitionCollector> partitionRequestMap;
private final IOperatorEnvironment env;
- private final Map<TaskId, ITaskState> taskStateMap;
+ private final Map<Object, IStateObject> stateObjectMap;
private final Map<TaskAttemptId, Task> taskMap;
@@ -75,25 +76,41 @@
private final IWorkspaceFileFactory fileFactory;
- private IJobletEventListener jobletEventListener;
+ private final Object globalJobData;
+
+ private final IJobletEventListener jobletEventListener;
+
+ private final int frameSize;
private JobStatus cleanupStatus;
private boolean cleanupPending;
- public Joblet(NodeControllerService nodeController, JobId jobId, INCApplicationContext appCtx, JobActivityGraph jag) {
+ public Joblet(NodeControllerService nodeController, JobId jobId, INCApplicationContext appCtx,
+ ActivityClusterGraph acg) {
this.nodeController = nodeController;
this.appCtx = appCtx;
this.jobId = jobId;
- this.jag = jag;
+ this.frameSize = acg.getFrameSize();
+ this.acg = acg;
partitionRequestMap = new HashMap<PartitionId, IPartitionCollector>();
env = new OperatorEnvironmentImpl(nodeController.getId());
- taskStateMap = new HashMap<TaskId, ITaskState>();
+ stateObjectMap = new HashMap<Object, IStateObject>();
taskMap = new HashMap<TaskAttemptId, Task>();
counterMap = new HashMap<String, Counter>();
deallocatableRegistry = new DefaultDeallocatableRegistry();
fileFactory = new WorkspaceFileFactory(this, (IOManager) appCtx.getRootContext().getIOManager());
cleanupPending = false;
+ IJobletEventListenerFactory jelf = acg.getJobletEventListenerFactory();
+ if (jelf != null) {
+ IJobletEventListener listener = jelf.createListener(this);
+ this.jobletEventListener = listener;
+ listener.jobletStart();
+ } else {
+ jobletEventListener = null;
+ }
+ IGlobalJobDataFactory gjdf = acg.getGlobalJobDataFactory();
+ globalJobData = gjdf != null ? gjdf.createGlobalJobData(this) : null;
}
@Override
@@ -101,8 +118,8 @@
return jobId;
}
- public JobActivityGraph getJobActivityGraph() {
- return jag;
+ public ActivityClusterGraph getActivityClusterGraph() {
+ return acg;
}
public IOperatorEnvironment getEnvironment() {
@@ -136,13 +153,13 @@
}
@Override
- public void setTaskState(ITaskState taskState) {
- taskStateMap.put(taskState.getTaskId(), taskState);
+ public synchronized void setStateObject(IStateObject taskState) {
+ stateObjectMap.put(taskState.getId(), taskState);
}
@Override
- public ITaskState getTaskState(TaskId taskId) {
- return taskStateMap.get(taskId);
+ public synchronized IStateObject getStateObject(Object id) {
+ return stateObjectMap.get(id);
}
}
@@ -187,18 +204,15 @@
});
}
- @Override
- public ByteBuffer allocateFrame() {
- return appCtx.getRootContext().allocateFrame();
+ ByteBuffer allocateFrame() {
+ return ByteBuffer.allocate(getFrameSize());
}
- @Override
- public int getFrameSize() {
- return appCtx.getRootContext().getFrameSize();
+ int getFrameSize() {
+ return frameSize;
}
- @Override
- public IIOManager getIOManager() {
+ IIOManager getIOManager() {
return appCtx.getRootContext().getIOManager();
}
@@ -222,6 +236,11 @@
return counter;
}
+ @Override
+ public Object getGlobalJobData() {
+ return globalJobData;
+ }
+
public synchronized void advertisePartitionRequest(TaskAttemptId taId, Collection<PartitionId> pids,
IPartitionCollector collector, PartitionState minState) throws Exception {
for (PartitionId pid : pids) {
@@ -242,10 +261,6 @@
return jobletEventListener;
}
- public void setJobletEventListener(IJobletEventListener jobletEventListener) {
- this.jobletEventListener = jobletEventListener;
- }
-
public void cleanup(JobStatus status) {
cleanupStatus = status;
cleanupPending = true;
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/NodeControllerService.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/NodeControllerService.java
index f3d00ac..58a173c 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/NodeControllerService.java
@@ -33,6 +33,7 @@
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -61,6 +62,7 @@
import edu.uci.ics.hyracks.control.nc.partitions.PartitionManager;
import edu.uci.ics.hyracks.control.nc.runtime.RootHyracksContext;
import edu.uci.ics.hyracks.control.nc.work.AbortTasksWork;
+import edu.uci.ics.hyracks.control.nc.work.ApplicationMessageWork;
import edu.uci.ics.hyracks.control.nc.work.BuildJobProfilesWork;
import edu.uci.ics.hyracks.control.nc.work.CleanupJobletWork;
import edu.uci.ics.hyracks.control.nc.work.CreateApplicationWork;
@@ -100,7 +102,7 @@
private final Map<JobId, Joblet> jobletMap;
- private final Executor executor;
+ private final ExecutorService executor;
private NodeParameters nodeParameters;
@@ -127,12 +129,12 @@
NodeControllerIPCI ipci = new NodeControllerIPCI();
ipc = new IPCSystem(new InetSocketAddress(ncConfig.clusterNetIPAddress, 0), ipci,
new CCNCFunctions.SerializerDeserializer());
- this.ctx = new RootHyracksContext(ncConfig.frameSize, new IOManager(getDevices(ncConfig.ioDevices), executor));
+ this.ctx = new RootHyracksContext(new IOManager(getDevices(ncConfig.ioDevices), executor));
if (id == null) {
throw new Exception("id not set");
}
partitionManager = new PartitionManager(this);
- netManager = new NetworkManager(ctx, getIpAddress(ncConfig), partitionManager, ncConfig.nNetThreads);
+ netManager = new NetworkManager(getIpAddress(ncConfig), partitionManager, ncConfig.nNetThreads);
queue = new WorkQueue();
jobletMap = new Hashtable<JobId, Joblet>();
@@ -214,6 +216,7 @@
@Override
public void stop() throws Exception {
LOGGER.log(Level.INFO, "Stopping NodeControllerService");
+ executor.shutdownNow();
partitionManager.close();
heartbeatTask.cancel();
netManager.stop();
@@ -257,7 +260,7 @@
return executor;
}
- public NCConfig getConfiguration() throws Exception {
+ public NCConfig getConfiguration() {
return ncConfig;
}
@@ -365,10 +368,16 @@
public void deliverIncomingMessage(IIPCHandle handle, long mid, long rmid, Object payload, Exception exception) {
CCNCFunctions.Function fn = (CCNCFunctions.Function) payload;
switch (fn.getFunctionId()) {
+ case SEND_APPLICATION_MESSAGE: {
+ CCNCFunctions.SendApplicationMessageFunction amf = (CCNCFunctions.SendApplicationMessageFunction) fn;
+ queue.schedule(new ApplicationMessageWork(NodeControllerService.this, amf.getMessage(), amf
+ .getAppName(), amf.getNodeId()));
+ return;
+ }
case START_TASKS: {
CCNCFunctions.StartTasksFunction stf = (CCNCFunctions.StartTasksFunction) fn;
queue.schedule(new StartTasksWork(NodeControllerService.this, stf.getAppName(), stf.getJobId(), stf
- .getPlanBytes(), stf.getTaskDescriptors(), stf.getConnectorPolicies()));
+ .getPlanBytes(), stf.getTaskDescriptors(), stf.getConnectorPolicies(), stf.getFlags()));
return;
}
@@ -414,4 +423,8 @@
}
}
+
+ public void sendApplicationMessageToCC(byte[] data, String appName, String nodeId) throws Exception {
+ ccs.sendApplicationMessageToCC(data, appName, nodeId);
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Task.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Task.java
index 1bb0b2f..eba3ec9 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Task.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Task.java
@@ -33,8 +33,7 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
-import edu.uci.ics.hyracks.api.dataflow.TaskId;
-import edu.uci.ics.hyracks.api.dataflow.state.ITaskState;
+import edu.uci.ics.hyracks.api.dataflow.state.IStateObject;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.api.io.FileReference;
@@ -88,7 +87,9 @@
private volatile boolean aborted;
- public Task(Joblet joblet, TaskAttemptId taskId, String displayName, Executor executor) {
+ private NodeControllerService ncs;
+
+ public Task(Joblet joblet, TaskAttemptId taskId, String displayName, Executor executor, NodeControllerService ncs) {
this.joblet = joblet;
this.taskAttemptId = taskId;
this.displayName = displayName;
@@ -102,6 +103,7 @@
failed = false;
errorBaos = new ByteArrayOutputStream();
errorWriter = new PrintWriter(errorBaos, true);
+ this.ncs = ncs;
}
public void setTaskRuntime(IPartitionCollector[] collectors, IOperatorNodePushable operator) {
@@ -306,19 +308,22 @@
reader.open();
try {
writer.open();
- ByteBuffer buffer = allocateFrame();
- while (reader.nextFrame(buffer)) {
- if (aborted) {
- return;
+ try {
+ ByteBuffer buffer = allocateFrame();
+ while (reader.nextFrame(buffer)) {
+ if (aborted) {
+ return;
+ }
+ buffer.flip();
+ writer.nextFrame(buffer);
+ buffer.compact();
}
- buffer.flip();
- writer.nextFrame(buffer);
- buffer.compact();
+ } catch (Exception e) {
+ writer.fail();
+ throw e;
+ } finally {
+ writer.close();
}
- writer.close();
- } catch (Exception e) {
- writer.fail();
- throw e;
} finally {
reader.close();
}
@@ -333,12 +338,18 @@
}
@Override
- public void setTaskState(ITaskState taskState) {
- opEnv.setTaskState(taskState);
+ public void setStateObject(IStateObject taskState) {
+ opEnv.setStateObject(taskState);
}
@Override
- public ITaskState getTaskState(TaskId taskId) {
- return opEnv.getTaskState(taskId);
+ public IStateObject getStateObject(Object id) {
+ return opEnv.getStateObject(id);
+ }
+
+ @Override
+ public void sendApplicationMessageToCC(byte[] message, String nodeId) throws Exception {
+ this.ncs.sendApplicationMessageToCC(message, this.getJobletContext().getApplicationContext()
+ .getApplicationName(), nodeId);
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/FileHandle.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/io/FileHandle.java
similarity index 87%
rename from hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/FileHandle.java
rename to hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/io/FileHandle.java
index 31e9821..8916fb1 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/FileHandle.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/io/FileHandle.java
@@ -12,13 +12,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.api.io;
+package edu.uci.ics.hyracks.control.nc.io;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
-public class FileHandle {
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IFileHandle;
+import edu.uci.ics.hyracks.api.io.IIOManager;
+
+public class FileHandle implements IFileHandle {
private final FileReference fileRef;
private RandomAccessFile raf;
@@ -79,4 +83,8 @@
public FileChannel getFileChannel() {
return channel;
}
+
+ public void sync(boolean metadata) throws IOException {
+ channel.force(metadata);
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/io/IOManager.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/io/IOManager.java
index c3105ab..3b13f32 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/io/IOManager.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/io/IOManager.java
@@ -24,8 +24,8 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.io.FileHandle;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IFileHandle;
import edu.uci.ics.hyracks.api.io.IIOFuture;
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
@@ -61,7 +61,7 @@
}
@Override
- public FileHandle open(FileReference fileRef, FileReadWriteMode rwMode, FileSyncMode syncMode)
+ public IFileHandle open(FileReference fileRef, FileReadWriteMode rwMode, FileSyncMode syncMode)
throws HyracksDataException {
FileHandle fHandle = new FileHandle(fileRef);
try {
@@ -73,14 +73,15 @@
}
@Override
- public int syncWrite(FileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException {
+ public int syncWrite(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException {
try {
int n = 0;
int remaining = data.remaining();
while (remaining > 0) {
- int len = fHandle.getFileChannel().write(data, offset);
+ int len = ((FileHandle) fHandle).getFileChannel().write(data, offset);
if (len < 0) {
- throw new HyracksDataException("Error writing to file: " + fHandle.getFileReference().toString());
+ throw new HyracksDataException("Error writing to file: "
+ + ((FileHandle) fHandle).getFileReference().toString());
}
remaining -= len;
offset += len;
@@ -95,12 +96,12 @@
}
@Override
- public int syncRead(FileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException {
+ public int syncRead(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException {
try {
int n = 0;
int remaining = data.remaining();
while (remaining > 0) {
- int len = fHandle.getFileChannel().read(data, offset);
+ int len = ((FileHandle) fHandle).getFileChannel().read(data, offset);
if (len < 0) {
return -1;
}
@@ -117,23 +118,23 @@
}
@Override
- public IIOFuture asyncWrite(FileHandle fHandle, long offset, ByteBuffer data) {
- AsyncWriteRequest req = new AsyncWriteRequest(fHandle, offset, data);
+ public IIOFuture asyncWrite(IFileHandle fHandle, long offset, ByteBuffer data) {
+ AsyncWriteRequest req = new AsyncWriteRequest((FileHandle) fHandle, offset, data);
executor.execute(req);
return req;
}
@Override
- public IIOFuture asyncRead(FileHandle fHandle, long offset, ByteBuffer data) {
- AsyncReadRequest req = new AsyncReadRequest(fHandle, offset, data);
+ public IIOFuture asyncRead(IFileHandle fHandle, long offset, ByteBuffer data) {
+ AsyncReadRequest req = new AsyncReadRequest((FileHandle) fHandle, offset, data);
executor.execute(req);
return req;
}
@Override
- public void close(FileHandle fHandle) throws HyracksDataException {
+ public void close(IFileHandle fHandle) throws HyracksDataException {
try {
- fHandle.close();
+ ((FileHandle) fHandle).close();
} catch (IOException e) {
throw new HyracksDataException(e);
}
@@ -225,4 +226,13 @@
return syncWrite(fHandle, offset, data);
}
}
+
+ @Override
+ public void sync(IFileHandle fileHandle, boolean metadata) throws HyracksDataException {
+ try {
+ ((FileHandle) fileHandle).sync(metadata);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkInputChannel.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkInputChannel.java
index 609d7f0..1d5af84 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkInputChannel.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkInputChannel.java
@@ -23,9 +23,8 @@
import edu.uci.ics.hyracks.api.channels.IInputChannel;
import edu.uci.ics.hyracks.api.channels.IInputChannelMonitor;
-import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
import edu.uci.ics.hyracks.net.buffers.IBufferAcceptor;
import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
@@ -34,8 +33,6 @@
public class NetworkInputChannel implements IInputChannel {
private static final Logger LOGGER = Logger.getLogger(NetworkInputChannel.class.getName());
- private IHyracksRootContext ctx;
-
private final NetworkManager netManager;
private final SocketAddress remoteAddress;
@@ -52,9 +49,8 @@
private Object attachment;
- public NetworkInputChannel(IHyracksRootContext ctx, NetworkManager netManager, SocketAddress remoteAddress,
- PartitionId partitionId, int nBuffers) {
- this.ctx = ctx;
+ public NetworkInputChannel(NetworkManager netManager, SocketAddress remoteAddress, PartitionId partitionId,
+ int nBuffers) {
this.netManager = netManager;
this.remoteAddress = remoteAddress;
this.partitionId = partitionId;
@@ -89,7 +85,7 @@
}
@Override
- public void open() throws HyracksDataException {
+ public void open(IHyracksTaskContext ctx) throws HyracksDataException {
try {
ccb = netManager.connect(remoteAddress);
} catch (Exception e) {
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkManager.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkManager.java
index 68e3120..b805595 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkManager.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkManager.java
@@ -23,12 +23,11 @@
import java.util.logging.Logger;
import edu.uci.ics.hyracks.api.comm.NetworkAddress;
-import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.control.nc.partitions.IPartitionRequestListener;
+import edu.uci.ics.hyracks.control.nc.partitions.PartitionManager;
import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
import edu.uci.ics.hyracks.net.exceptions.NetException;
import edu.uci.ics.hyracks.net.protocols.muxdemux.ChannelControlBlock;
@@ -40,21 +39,20 @@
public class NetworkManager {
private static final Logger LOGGER = Logger.getLogger(NetworkManager.class.getName());
+ private static final int MAX_CONNECTION_ATTEMPTS = 5;
+
static final int INITIAL_MESSAGE_SIZE = 20;
- private final IHyracksRootContext ctx;
-
- private final IPartitionRequestListener partitionRequestListener;
+ private final PartitionManager partitionManager;
private final MuxDemux md;
private NetworkAddress networkAddress;
- public NetworkManager(IHyracksRootContext ctx, InetAddress inetAddress,
- IPartitionRequestListener partitionRequestListener, int nThreads) throws IOException {
- this.ctx = ctx;
- this.partitionRequestListener = partitionRequestListener;
- md = new MuxDemux(new InetSocketAddress(inetAddress, 0), new ChannelOpenListener(), nThreads);
+ public NetworkManager(InetAddress inetAddress, PartitionManager partitionManager, int nThreads) throws IOException {
+ this.partitionManager = partitionManager;
+ md = new MuxDemux(new InetSocketAddress(inetAddress, 0), new ChannelOpenListener(), nThreads,
+ MAX_CONNECTION_ATTEMPTS);
}
public void start() throws IOException {
@@ -99,9 +97,9 @@
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("Received initial partition request: " + pid + " on channel: " + ccb);
}
- noc = new NetworkOutputChannel(ctx, ccb, 5);
+ noc = new NetworkOutputChannel(ccb, 1);
try {
- partitionRequestListener.registerPartitionRequest(pid, noc);
+ partitionManager.registerPartitionRequest(pid, noc);
} catch (HyracksException e) {
noc.abort();
}
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkOutputChannel.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkOutputChannel.java
index 185768e..9024e18 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkOutputChannel.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkOutputChannel.java
@@ -19,7 +19,7 @@
import java.util.Deque;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.net.buffers.IBufferAcceptor;
import edu.uci.ics.hyracks.net.protocols.muxdemux.ChannelControlBlock;
@@ -27,17 +27,23 @@
public class NetworkOutputChannel implements IFrameWriter {
private final ChannelControlBlock ccb;
+ private final int nBuffers;
+
private final Deque<ByteBuffer> emptyStack;
private boolean aborted;
- public NetworkOutputChannel(IHyracksRootContext ctx, ChannelControlBlock ccb, int nBuffers) {
+ public NetworkOutputChannel(ChannelControlBlock ccb, int nBuffers) {
this.ccb = ccb;
+ this.nBuffers = nBuffers;
emptyStack = new ArrayDeque<ByteBuffer>(nBuffers);
+ ccb.getWriteInterface().setEmptyBufferAcceptor(new WriteEmptyBufferAcceptor());
+ }
+
+ public void setTaskContext(IHyracksTaskContext ctx) {
for (int i = 0; i < nBuffers; ++i) {
emptyStack.push(ByteBuffer.allocateDirect(ctx.getFrameSize()));
}
- ccb.getWriteInterface().setEmptyBufferAcceptor(new WriteEmptyBufferAcceptor());
}
@Override
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/IPartitionRequestListener.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/IPartitionRequestListener.java
deleted file mode 100644
index 20ed49c..0000000
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/IPartitionRequestListener.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.nc.partitions;
-
-import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.partitions.PartitionId;
-
-public interface IPartitionRequestListener {
- public void registerPartitionRequest(PartitionId partitionId, IFrameWriter writer) throws HyracksException;
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartition.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartition.java
index 97f82d9..ae4fd2b 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartition.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartition.java
@@ -18,16 +18,16 @@
import java.util.concurrent.Executor;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileHandle;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IFileHandle;
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.api.partitions.IPartition;
import edu.uci.ics.hyracks.control.nc.io.IOManager;
public class MaterializedPartition implements IPartition {
- private final IHyracksRootContext ctx;
+ private final IHyracksTaskContext ctx;
private final FileReference partitionFile;
@@ -35,7 +35,7 @@
private final IOManager ioManager;
- public MaterializedPartition(IHyracksRootContext ctx, FileReference partitionFile, Executor executor,
+ public MaterializedPartition(IHyracksTaskContext ctx, FileReference partitionFile, Executor executor,
IOManager ioManager) {
this.ctx = ctx;
this.partitionFile = partitionFile;
@@ -44,6 +44,11 @@
}
@Override
+ public IHyracksTaskContext getTaskContext() {
+ return ctx;
+ }
+
+ @Override
public void deallocate() {
partitionFile.delete();
}
@@ -54,7 +59,7 @@
@Override
public void run() {
try {
- FileHandle fh = ioManager.open(partitionFile, IIOManager.FileReadWriteMode.READ_ONLY,
+ IFileHandle fh = ioManager.open(partitionFile, IIOManager.FileReadWriteMode.READ_ONLY,
IIOManager.FileSyncMode.METADATA_ASYNC_DATA_ASYNC);
try {
writer.open();
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartitionInputChannel.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartitionInputChannel.java
index 0ea3124..16e31f7 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartitionInputChannel.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartitionInputChannel.java
@@ -21,12 +21,14 @@
import edu.uci.ics.hyracks.api.channels.IInputChannel;
import edu.uci.ics.hyracks.api.channels.IInputChannelMonitor;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.partitions.IPartition;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
public class MaterializedPartitionInputChannel implements IInputChannel {
+ private final int nBuffers;
+
private final Queue<ByteBuffer> emptyQueue;
private final Queue<ByteBuffer> fullQueue;
@@ -41,12 +43,9 @@
private Object attachment;
- public MaterializedPartitionInputChannel(IHyracksRootContext ctx, int nBuffers, PartitionId pid,
- PartitionManager manager) {
+ public MaterializedPartitionInputChannel(int nBuffers, PartitionId pid, PartitionManager manager) {
+ this.nBuffers = nBuffers;
this.emptyQueue = new ArrayDeque<ByteBuffer>(nBuffers);
- for (int i = 0; i < nBuffers; ++i) {
- emptyQueue.add(ctx.allocateFrame());
- }
fullQueue = new ArrayDeque<ByteBuffer>(nBuffers);
this.pid = pid;
this.manager = manager;
@@ -83,7 +82,10 @@
}
@Override
- public void open() throws HyracksDataException {
+ public void open(IHyracksTaskContext ctx) throws HyracksDataException {
+ for (int i = 0; i < nBuffers; ++i) {
+ emptyQueue.add(ctx.allocateFrame());
+ }
IPartition partition = manager.getPartition(pid);
partition.writeTo(writer);
}
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartitionWriter.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartitionWriter.java
index 1a1b15f..7bd0eb1 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartitionWriter.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartitionWriter.java
@@ -20,11 +20,11 @@
import java.util.logging.Logger;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileHandle;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IFileHandle;
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
import edu.uci.ics.hyracks.control.common.job.PartitionState;
@@ -33,7 +33,7 @@
public class MaterializedPartitionWriter implements IFrameWriter {
private static final Logger LOGGER = Logger.getLogger(MaterializedPartitionWriter.class.getName());
- private final IHyracksRootContext ctx;
+ private final IHyracksTaskContext ctx;
private final PartitionManager manager;
@@ -45,13 +45,13 @@
private FileReference fRef;
- private FileHandle handle;
+ private IFileHandle handle;
private long size;
private boolean failed;
- public MaterializedPartitionWriter(IHyracksRootContext ctx, PartitionManager manager, PartitionId pid,
+ public MaterializedPartitionWriter(IHyracksTaskContext ctx, PartitionManager manager, PartitionId pid,
TaskAttemptId taId, Executor executor) {
this.ctx = ctx;
this.manager = manager;
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java
index 143fd2c..62320c5 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java
@@ -20,11 +20,11 @@
import java.util.logging.Logger;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileHandle;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IFileHandle;
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.api.partitions.IPartition;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
@@ -34,7 +34,7 @@
public class MaterializingPipelinedPartition implements IFrameWriter, IPartition {
private static final Logger LOGGER = Logger.getLogger(MaterializingPipelinedPartition.class.getName());
- private final IHyracksRootContext ctx;
+ private final IHyracksTaskContext ctx;
private final Executor executor;
@@ -48,7 +48,7 @@
private FileReference fRef;
- private FileHandle handle;
+ private IFileHandle handle;
private long size;
@@ -56,7 +56,7 @@
private boolean failed;
- public MaterializingPipelinedPartition(IHyracksRootContext ctx, PartitionManager manager, PartitionId pid,
+ public MaterializingPipelinedPartition(IHyracksTaskContext ctx, PartitionManager manager, PartitionId pid,
TaskAttemptId taId, Executor executor) {
this.ctx = ctx;
this.executor = executor;
@@ -67,6 +67,11 @@
}
@Override
+ public IHyracksTaskContext getTaskContext() {
+ return ctx;
+ }
+
+ @Override
public void deallocate() {
fRef.delete();
}
@@ -77,7 +82,7 @@
@Override
public void run() {
try {
- FileHandle fh = ioManager.open(fRef, IIOManager.FileReadWriteMode.READ_ONLY,
+ IFileHandle fh = ioManager.open(fRef, IIOManager.FileReadWriteMode.READ_ONLY,
IIOManager.FileSyncMode.METADATA_ASYNC_DATA_ASYNC);
try {
writer.open();
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PartitionManager.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PartitionManager.java
index ce40fe6..45c091a 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PartitionManager.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PartitionManager.java
@@ -21,7 +21,6 @@
import java.util.List;
import java.util.Map;
-import edu.uci.ics.hyracks.api.comm.IFrameWriter;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
@@ -34,9 +33,10 @@
import edu.uci.ics.hyracks.control.nc.NodeControllerService;
import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.control.nc.io.WorkspaceFileFactory;
+import edu.uci.ics.hyracks.control.nc.net.NetworkOutputChannel;
import edu.uci.ics.hyracks.control.nc.resources.DefaultDeallocatableRegistry;
-public class PartitionManager implements IPartitionRequestListener {
+public class PartitionManager {
private final NodeControllerService ncs;
private final Map<PartitionId, List<IPartition>> partitionMap;
@@ -93,12 +93,12 @@
}
}
- @Override
- public synchronized void registerPartitionRequest(PartitionId partitionId, IFrameWriter writer)
+ public synchronized void registerPartitionRequest(PartitionId partitionId, NetworkOutputChannel writer)
throws HyracksException {
List<IPartition> pList = partitionMap.get(partitionId);
if (pList != null && !pList.isEmpty()) {
IPartition partition = pList.get(0);
+ writer.setTaskContext(partition.getTaskContext());
partition.writeTo(writer);
if (!partition.isReusable()) {
partitionMap.remove(partitionId);
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PipelinedPartition.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PipelinedPartition.java
index 92dc0b2..e427cf3 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PipelinedPartition.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PipelinedPartition.java
@@ -17,6 +17,7 @@
import java.nio.ByteBuffer;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.partitions.IPartition;
@@ -24,6 +25,8 @@
import edu.uci.ics.hyracks.control.common.job.PartitionState;
public class PipelinedPartition implements IFrameWriter, IPartition {
+ private final IHyracksTaskContext ctx;
+
private final PartitionManager manager;
private final PartitionId pid;
@@ -36,13 +39,19 @@
private boolean failed;
- public PipelinedPartition(PartitionManager manager, PartitionId pid, TaskAttemptId taId) {
+ public PipelinedPartition(IHyracksTaskContext ctx, PartitionManager manager, PartitionId pid, TaskAttemptId taId) {
+ this.ctx = ctx;
this.manager = manager;
this.pid = pid;
this.taId = taId;
}
@Override
+ public IHyracksTaskContext getTaskContext() {
+ return ctx;
+ }
+
+ @Override
public boolean isReusable() {
return false;
}
@@ -89,7 +98,6 @@
@Override
public void fail() throws HyracksDataException {
- ensureConnected();
failed = true;
if (delegate != null) {
delegate.fail();
@@ -98,10 +106,10 @@
@Override
public void close() throws HyracksDataException {
- ensureConnected();
if (!failed) {
+ ensureConnected();
manager.updatePartitionState(pid, taId, this, PartitionState.COMMITTED);
+ delegate.close();
}
- delegate.close();
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/ReceiveSideMaterializingCollector.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/ReceiveSideMaterializingCollector.java
index f93f874..b1e58fc 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/ReceiveSideMaterializingCollector.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/ReceiveSideMaterializingCollector.java
@@ -18,13 +18,15 @@
import java.util.Collection;
import java.util.Collections;
import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import edu.uci.ics.hyracks.api.channels.IInputChannel;
import edu.uci.ics.hyracks.api.channels.IInputChannelMonitor;
import edu.uci.ics.hyracks.api.comm.IFrameReader;
import edu.uci.ics.hyracks.api.comm.IPartitionCollector;
import edu.uci.ics.hyracks.api.comm.PartitionChannel;
-import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
@@ -33,7 +35,7 @@
import edu.uci.ics.hyracks.api.partitions.PartitionId;
public class ReceiveSideMaterializingCollector implements IPartitionCollector {
- private final IHyracksRootContext ctx;
+ private final IHyracksTaskContext ctx;
private PartitionManager manager;
@@ -43,7 +45,7 @@
private final Executor executor;
- public ReceiveSideMaterializingCollector(IHyracksRootContext ctx, PartitionManager manager,
+ public ReceiveSideMaterializingCollector(IHyracksTaskContext ctx, PartitionManager manager,
IPartitionCollector collector, TaskAttemptId taId, Executor executor) {
this.ctx = ctx;
this.manager = manager;
@@ -83,58 +85,63 @@
private class PartitionWriter implements Runnable, IInputChannelMonitor {
private PartitionChannel pc;
- private int nAvailableFrames;
+ private final AtomicInteger nAvailableFrames;
- private boolean eos;
+ private final AtomicBoolean eos;
- private boolean failed;
+ private final AtomicBoolean failed;
public PartitionWriter(PartitionChannel pc) {
this.pc = pc;
- nAvailableFrames = 0;
- eos = false;
+ nAvailableFrames = new AtomicInteger(0);
+ eos = new AtomicBoolean(false);
+ failed = new AtomicBoolean(false);
}
@Override
public synchronized void notifyFailure(IInputChannel channel) {
- failed = true;
+ failed.set(true);
notifyAll();
}
@Override
public synchronized void notifyDataAvailability(IInputChannel channel, int nFrames) {
- nAvailableFrames += nFrames;
+ nAvailableFrames.addAndGet(nFrames);
notifyAll();
}
@Override
public synchronized void notifyEndOfStream(IInputChannel channel) {
- eos = true;
+ eos.set(true);
notifyAll();
}
@Override
- public synchronized void run() {
+ public void run() {
PartitionId pid = pc.getPartitionId();
MaterializedPartitionWriter mpw = new MaterializedPartitionWriter(ctx, manager, pid, taId, executor);
IInputChannel channel = pc.getInputChannel();
try {
channel.registerMonitor(this);
- channel.open();
+ channel.open(ctx);
mpw.open();
while (true) {
- if (nAvailableFrames > 0) {
+ if (nAvailableFrames.get() > 0) {
ByteBuffer buffer = channel.getNextBuffer();
- --nAvailableFrames;
+ nAvailableFrames.decrementAndGet();
mpw.nextFrame(buffer);
channel.recycleBuffer(buffer);
- } else if (eos) {
+ } else if (eos.get()) {
break;
- } else if (failed) {
+ } else if (failed.get()) {
throw new HyracksDataException("Failure occurred on input");
} else {
try {
- wait();
+ synchronized (this) {
+ if (nAvailableFrames.get() <= 0 && !eos.get() && !failed.get()) {
+ wait();
+ }
+ }
} catch (InterruptedException e) {
throw new HyracksDataException(e);
}
@@ -143,7 +150,7 @@
mpw.close();
channel.close();
delegate.addPartitions(Collections.singleton(new PartitionChannel(pid,
- new MaterializedPartitionInputChannel(ctx, 5, pid, manager))));
+ new MaterializedPartitionInputChannel(1, pid, manager))));
} catch (HyracksException e) {
}
}
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/runtime/RootHyracksContext.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/runtime/RootHyracksContext.java
index 045fa52..9e42277 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/runtime/RootHyracksContext.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/runtime/RootHyracksContext.java
@@ -14,33 +14,18 @@
*/
package edu.uci.ics.hyracks.control.nc.runtime;
-import java.nio.ByteBuffer;
-
import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
import edu.uci.ics.hyracks.api.io.IIOManager;
public class RootHyracksContext implements IHyracksRootContext {
- private final int frameSize;
-
private final IIOManager ioManager;
- public RootHyracksContext(int frameSize, IIOManager ioManager) {
- this.frameSize = frameSize;
+ public RootHyracksContext(IIOManager ioManager) {
this.ioManager = ioManager;
}
@Override
- public int getFrameSize() {
- return frameSize;
- }
-
- @Override
public IIOManager getIOManager() {
return ioManager;
}
-
- @Override
- public ByteBuffer allocateFrame() {
- return ByteBuffer.allocate(frameSize);
- }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/AbortTasksWork.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/AbortTasksWork.java
index e8c4052..8f8c032 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/AbortTasksWork.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/AbortTasksWork.java
@@ -21,12 +21,12 @@
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
import edu.uci.ics.hyracks.control.nc.Joblet;
import edu.uci.ics.hyracks.control.nc.NodeControllerService;
import edu.uci.ics.hyracks.control.nc.Task;
-public class AbortTasksWork extends SynchronizableWork {
+public class AbortTasksWork extends AbstractWork {
private static final Logger LOGGER = Logger.getLogger(AbortTasksWork.class.getName());
private final NodeControllerService ncs;
@@ -42,7 +42,7 @@
}
@Override
- protected void doRun() throws Exception {
+ public void run() {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Aborting Tasks: " + jobId + ":" + tasks);
}
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/ApplicationMessageWork.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/ApplicationMessageWork.java
new file mode 100644
index 0000000..deb1b75
--- /dev/null
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/ApplicationMessageWork.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.nc.work;
+
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.messages.IMessage;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+import edu.uci.ics.hyracks.control.nc.NodeControllerService;
+import edu.uci.ics.hyracks.control.nc.application.NCApplicationContext;
+
+/**
+ * @author rico
+ *
+ */
+public class ApplicationMessageWork extends AbstractWork {
+
+ private static final Logger LOGGER = Logger.getLogger(ApplicationMessageWork.class.getName());
+ private byte[] message;
+ private String nodeId;
+ private NodeControllerService ncs;
+ private String appName;
+
+ public ApplicationMessageWork(NodeControllerService ncs, byte[] message, String appName, String nodeId) {
+ this.ncs = ncs;
+ this.nodeId = nodeId;
+ this.message = message;
+ this.appName = appName;
+ }
+
+ @Override
+ public void run() {
+
+ NCApplicationContext ctx = ncs.getApplications().get(appName);
+ try {
+ IMessage data = (IMessage) ctx.deserialize(message);
+ if (ctx.getMessageBroker() != null) {
+ ctx.getMessageBroker().receivedMessage(data, nodeId);
+ } else {
+ LOGGER.log(Level.WARNING, "Messsage was sent, but no Message Broker set!");
+ }
+ } catch (IOException e) {
+ Logger.getLogger(this.getClass().getName()).log(Level.WARNING, "Error in application message delivery!", e);
+ } catch (ClassNotFoundException e) {
+ Logger.getLogger(this.getClass().getName()).log(Level.WARNING, "Error in application message delivery!", e);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "nodeID: " + nodeId;
+ }
+
+}
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CleanupJobletWork.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CleanupJobletWork.java
index b75a1fc..173ab92 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CleanupJobletWork.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CleanupJobletWork.java
@@ -23,11 +23,11 @@
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobStatus;
import edu.uci.ics.hyracks.api.partitions.IPartition;
-import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
import edu.uci.ics.hyracks.control.nc.Joblet;
import edu.uci.ics.hyracks.control.nc.NodeControllerService;
-public class CleanupJobletWork extends SynchronizableWork {
+public class CleanupJobletWork extends AbstractWork {
private static final Logger LOGGER = Logger.getLogger(CleanupJobletWork.class.getName());
private final NodeControllerService ncs;
@@ -43,7 +43,7 @@
}
@Override
- protected void doRun() throws Exception {
+ public void run() {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Cleaning up after job: " + jobId);
}
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CreateApplicationWork.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CreateApplicationWork.java
index eb982df..6eb1a95 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CreateApplicationWork.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CreateApplicationWork.java
@@ -18,7 +18,6 @@
import java.io.OutputStream;
import java.io.Serializable;
import java.util.Map;
-import java.util.logging.Logger;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
@@ -30,13 +29,11 @@
import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
import edu.uci.ics.hyracks.control.common.controllers.NCConfig;
import edu.uci.ics.hyracks.control.common.controllers.NodeParameters;
-import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
import edu.uci.ics.hyracks.control.nc.NodeControllerService;
import edu.uci.ics.hyracks.control.nc.application.NCApplicationContext;
-public class CreateApplicationWork extends SynchronizableWork {
- private static final Logger LOGGER = Logger.getLogger(CreateApplicationWork.class.getName());
-
+public class CreateApplicationWork extends AbstractWork {
private final NodeControllerService ncs;
private final String appName;
@@ -54,7 +51,7 @@
}
@Override
- protected void doRun() throws Exception {
+ public void run() {
try {
NCApplicationContext appCtx;
Map<String, NCApplicationContext> applications = ncs.getApplications();
@@ -85,7 +82,7 @@
ncs.getClusterController()
.notifyApplicationStateChange(ncs.getId(), appName, ApplicationStatus.INITIALIZED);
} catch (Exception e) {
- LOGGER.warning("Error creating application: " + e.getMessage());
+ throw new RuntimeException(e);
}
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/DestroyApplicationWork.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/DestroyApplicationWork.java
index cfe00f6..b104ce8 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/DestroyApplicationWork.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/DestroyApplicationWork.java
@@ -15,17 +15,14 @@
package edu.uci.ics.hyracks.control.nc.work;
import java.util.Map;
-import java.util.logging.Logger;
import edu.uci.ics.hyracks.control.common.application.ApplicationContext;
import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
-import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
import edu.uci.ics.hyracks.control.nc.NodeControllerService;
import edu.uci.ics.hyracks.control.nc.application.NCApplicationContext;
-public class DestroyApplicationWork extends SynchronizableWork {
- private static final Logger LOGGER = Logger.getLogger(DestroyApplicationWork.class.getName());
-
+public class DestroyApplicationWork extends AbstractWork {
private final NodeControllerService ncs;
private final String appName;
@@ -36,16 +33,17 @@
}
@Override
- protected void doRun() throws Exception {
+ public void run() {
try {
Map<String, NCApplicationContext> applications = ncs.getApplications();
ApplicationContext appCtx = applications.remove(appName);
if (appCtx != null) {
appCtx.deinitialize();
}
+ ncs.getClusterController().notifyApplicationStateChange(ncs.getId(), appName,
+ ApplicationStatus.DEINITIALIZED);
} catch (Exception e) {
- LOGGER.warning("Error destroying application: " + e.getMessage());
+ throw new RuntimeException(e);
}
- ncs.getClusterController().notifyApplicationStateChange(ncs.getId(), appName, ApplicationStatus.DEINITIALIZED);
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/ReportPartitionAvailabilityWork.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/ReportPartitionAvailabilityWork.java
index 9734567..bb9669d 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/ReportPartitionAvailabilityWork.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/ReportPartitionAvailabilityWork.java
@@ -17,20 +17,17 @@
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.util.Map;
-import java.util.logging.Logger;
import edu.uci.ics.hyracks.api.comm.NetworkAddress;
import edu.uci.ics.hyracks.api.comm.PartitionChannel;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
import edu.uci.ics.hyracks.control.nc.Joblet;
import edu.uci.ics.hyracks.control.nc.NodeControllerService;
import edu.uci.ics.hyracks.control.nc.net.NetworkInputChannel;
-public class ReportPartitionAvailabilityWork extends SynchronizableWork {
- private static final Logger LOGGER = Logger.getLogger(ReportPartitionAvailabilityWork.class.getName());
-
+public class ReportPartitionAvailabilityWork extends AbstractWork {
private final NodeControllerService ncs;
private final PartitionId pid;
@@ -44,14 +41,18 @@
}
@Override
- protected void doRun() throws Exception {
- Map<JobId, Joblet> jobletMap = ncs.getJobletMap();
- Joblet ji = jobletMap.get(pid.getJobId());
- if (ji != null) {
- PartitionChannel channel = new PartitionChannel(pid, new NetworkInputChannel(ncs.getRootContext(),
- ncs.getNetworkManager(), new InetSocketAddress(InetAddress.getByAddress(networkAddress
- .getIpAddress()), networkAddress.getPort()), pid, 5));
- ji.reportPartitionAvailability(channel);
+ public void run() {
+ try {
+ Map<JobId, Joblet> jobletMap = ncs.getJobletMap();
+ Joblet ji = jobletMap.get(pid.getJobId());
+ if (ji != null) {
+ PartitionChannel channel = new PartitionChannel(pid, new NetworkInputChannel(ncs.getNetworkManager(),
+ new InetSocketAddress(InetAddress.getByAddress(networkAddress.getIpAddress()),
+ networkAddress.getPort()), pid, 5));
+ ji.reportPartitionAvailability(channel);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
}
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/StartTasksWork.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/StartTasksWork.java
index a101612..0c0fa3d 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/StartTasksWork.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/StartTasksWork.java
@@ -26,25 +26,24 @@
import edu.uci.ics.hyracks.api.comm.IPartitionCollector;
import edu.uci.ics.hyracks.api.comm.IPartitionWriterFactory;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.ActivityId;
import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
import edu.uci.ics.hyracks.api.dataflow.IActivity;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
-import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.dataflow.TaskId;
import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IJobletEventListener;
-import edu.uci.ics.hyracks.api.job.IJobletEventListenerFactory;
-import edu.uci.ics.hyracks.api.job.JobActivityGraph;
+import edu.uci.ics.hyracks.api.job.ActivityCluster;
+import edu.uci.ics.hyracks.api.job.ActivityClusterGraph;
import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
import edu.uci.ics.hyracks.control.common.job.TaskAttemptDescriptor;
-import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
import edu.uci.ics.hyracks.control.nc.Joblet;
import edu.uci.ics.hyracks.control.nc.NodeControllerService;
import edu.uci.ics.hyracks.control.nc.Task;
@@ -55,7 +54,7 @@
import edu.uci.ics.hyracks.control.nc.partitions.ReceiveSideMaterializingCollector;
import edu.uci.ics.hyracks.control.nc.profiling.ProfilingPartitionWriterFactory;
-public class StartTasksWork extends SynchronizableWork {
+public class StartTasksWork extends AbstractWork {
private static final Logger LOGGER = Logger.getLogger(StartTasksWork.class.getName());
private final NodeControllerService ncs;
@@ -64,58 +63,67 @@
private final JobId jobId;
- private final byte[] jagBytes;
+ private final byte[] acgBytes;
private final List<TaskAttemptDescriptor> taskDescriptors;
private final Map<ConnectorDescriptorId, IConnectorPolicy> connectorPoliciesMap;
- public StartTasksWork(NodeControllerService ncs, String appName, JobId jobId, byte[] jagBytes,
+ private final EnumSet<JobFlag> flags;
+
+ public StartTasksWork(NodeControllerService ncs, String appName, JobId jobId, byte[] acgBytes,
List<TaskAttemptDescriptor> taskDescriptors,
- Map<ConnectorDescriptorId, IConnectorPolicy> connectorPoliciesMap) {
+ Map<ConnectorDescriptorId, IConnectorPolicy> connectorPoliciesMap, EnumSet<JobFlag> flags) {
this.ncs = ncs;
this.appName = appName;
this.jobId = jobId;
- this.jagBytes = jagBytes;
+ this.acgBytes = acgBytes;
this.taskDescriptors = taskDescriptors;
this.connectorPoliciesMap = connectorPoliciesMap;
+ this.flags = flags;
}
@Override
- protected void doRun() throws Exception {
+ public void run() {
try {
Map<String, NCApplicationContext> applications = ncs.getApplications();
NCApplicationContext appCtx = applications.get(appName);
- final Joblet joblet = getOrCreateLocalJoblet(jobId, appCtx, jagBytes == null ? null
- : (JobActivityGraph) appCtx.deserialize(jagBytes));
- final JobActivityGraph jag = joblet.getJobActivityGraph();
+ final Joblet joblet = getOrCreateLocalJoblet(jobId, appCtx, acgBytes == null ? null
+ : (ActivityClusterGraph) appCtx.deserialize(acgBytes));
+ final ActivityClusterGraph acg = joblet.getActivityClusterGraph();
IRecordDescriptorProvider rdp = new IRecordDescriptorProvider() {
@Override
- public RecordDescriptor getOutputRecordDescriptor(OperatorDescriptorId opId, int outputIndex) {
- return jag.getJobSpecification().getOperatorOutputRecordDescriptor(opId, outputIndex);
+ public RecordDescriptor getOutputRecordDescriptor(ActivityId aid, int outputIndex) {
+ ActivityCluster ac = acg.getActivityMap().get(aid);
+ IConnectorDescriptor conn = ac.getActivityOutputMap().get(aid).get(outputIndex);
+ return ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId());
}
@Override
- public RecordDescriptor getInputRecordDescriptor(OperatorDescriptorId opId, int inputIndex) {
- return jag.getJobSpecification().getOperatorInputRecordDescriptor(opId, inputIndex);
+ public RecordDescriptor getInputRecordDescriptor(ActivityId aid, int inputIndex) {
+ ActivityCluster ac = acg.getActivityMap().get(aid);
+ IConnectorDescriptor conn = ac.getActivityInputMap().get(aid).get(inputIndex);
+ return ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId());
}
};
for (TaskAttemptDescriptor td : taskDescriptors) {
TaskAttemptId taId = td.getTaskAttemptId();
TaskId tid = taId.getTaskId();
- IActivity han = jag.getActivityNodeMap().get(tid.getActivityId());
+ ActivityId aid = tid.getActivityId();
+ ActivityCluster ac = acg.getActivityMap().get(aid);
+ IActivity han = ac.getActivityMap().get(aid);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Initializing " + taId + " -> " + han);
}
final int partition = tid.getPartition();
- Task task = new Task(joblet, taId, han.getClass().getName(), ncs.getExecutor());
+ Task task = new Task(joblet, taId, han.getClass().getName(), ncs.getExecutor(), ncs);
IOperatorNodePushable operator = han.createPushRuntime(task, rdp, partition, td.getPartitionCount());
List<IPartitionCollector> collectors = new ArrayList<IPartitionCollector>();
- List<IConnectorDescriptor> inputs = jag.getActivityInputConnectorDescriptors(tid.getActivityId());
+ List<IConnectorDescriptor> inputs = ac.getActivityInputMap().get(aid);
if (inputs != null) {
for (int i = 0; i < inputs.size(); ++i) {
IConnectorDescriptor conn = inputs.get(i);
@@ -123,21 +131,21 @@
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("input: " + i + ": " + conn.getConnectorId());
}
- RecordDescriptor recordDesc = jag.getJobSpecification().getConnectorRecordDescriptor(conn);
+ RecordDescriptor recordDesc = ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId());
IPartitionCollector collector = createPartitionCollector(td, partition, task, i, conn,
recordDesc, cPolicy);
collectors.add(collector);
}
}
- List<IConnectorDescriptor> outputs = jag.getActivityOutputConnectorDescriptors(tid.getActivityId());
+ List<IConnectorDescriptor> outputs = ac.getActivityOutputMap().get(aid);
if (outputs != null) {
for (int i = 0; i < outputs.size(); ++i) {
final IConnectorDescriptor conn = outputs.get(i);
- RecordDescriptor recordDesc = jag.getJobSpecification().getConnectorRecordDescriptor(conn);
+ RecordDescriptor recordDesc = ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId());
IConnectorPolicy cPolicy = connectorPoliciesMap.get(conn.getConnectorId());
IPartitionWriterFactory pwFactory = createPartitionWriterFactory(task, cPolicy, jobId, conn,
- partition, taId, jag.getJobFlags());
+ partition, taId, flags);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("output: " + i + ": " + conn.getConnectorId());
@@ -155,25 +163,19 @@
}
} catch (Exception e) {
e.printStackTrace();
- throw e;
+ throw new RuntimeException(e);
}
}
- private Joblet getOrCreateLocalJoblet(JobId jobId, INCApplicationContext appCtx, JobActivityGraph jag)
+ private Joblet getOrCreateLocalJoblet(JobId jobId, INCApplicationContext appCtx, ActivityClusterGraph acg)
throws Exception {
Map<JobId, Joblet> jobletMap = ncs.getJobletMap();
Joblet ji = jobletMap.get(jobId);
if (ji == null) {
- if (jag == null) {
+ if (acg == null) {
throw new NullPointerException("JobActivityGraph was null");
}
- ji = new Joblet(ncs, jobId, appCtx, jag);
- IJobletEventListenerFactory jelf = jag.getJobSpecification().getJobletEventListenerFactory();
- if (jelf != null) {
- IJobletEventListener listener = jelf.createListener(ji);
- ji.setJobletEventListener(listener);
- listener.jobletStart();
- }
+ ji = new Joblet(ncs, jobId, appCtx, acg);
jobletMap.put(jobId, ji);
}
return ji;
@@ -185,34 +187,32 @@
IPartitionCollector collector = conn.createPartitionCollector(task, recordDesc, partition,
td.getInputPartitionCounts()[i], td.getPartitionCount());
if (cPolicy.materializeOnReceiveSide()) {
- return new ReceiveSideMaterializingCollector(ncs.getRootContext(), ncs.getPartitionManager(), collector,
+ return new ReceiveSideMaterializingCollector(task, ncs.getPartitionManager(), collector,
task.getTaskAttemptId(), ncs.getExecutor());
} else {
return collector;
}
}
- private IPartitionWriterFactory createPartitionWriterFactory(IHyracksTaskContext ctx, IConnectorPolicy cPolicy,
- final JobId jobId, final IConnectorDescriptor conn, final int senderIndex, final TaskAttemptId taId,
- EnumSet<JobFlag> flags) {
+ private IPartitionWriterFactory createPartitionWriterFactory(final IHyracksTaskContext ctx,
+ IConnectorPolicy cPolicy, final JobId jobId, final IConnectorDescriptor conn, final int senderIndex,
+ final TaskAttemptId taId, EnumSet<JobFlag> flags) {
IPartitionWriterFactory factory;
if (cPolicy.materializeOnSendSide()) {
if (cPolicy.consumerWaitsForProducerToFinish()) {
factory = new IPartitionWriterFactory() {
@Override
public IFrameWriter createFrameWriter(int receiverIndex) throws HyracksDataException {
- return new MaterializedPartitionWriter(ncs.getRootContext(), ncs.getPartitionManager(),
- new PartitionId(jobId, conn.getConnectorId(), senderIndex, receiverIndex), taId,
- ncs.getExecutor());
+ return new MaterializedPartitionWriter(ctx, ncs.getPartitionManager(), new PartitionId(jobId,
+ conn.getConnectorId(), senderIndex, receiverIndex), taId, ncs.getExecutor());
}
};
} else {
factory = new IPartitionWriterFactory() {
@Override
public IFrameWriter createFrameWriter(int receiverIndex) throws HyracksDataException {
- return new MaterializingPipelinedPartition(ncs.getRootContext(), ncs.getPartitionManager(),
- new PartitionId(jobId, conn.getConnectorId(), senderIndex, receiverIndex), taId,
- ncs.getExecutor());
+ return new MaterializingPipelinedPartition(ctx, ncs.getPartitionManager(), new PartitionId(
+ jobId, conn.getConnectorId(), senderIndex, receiverIndex), taId, ncs.getExecutor());
}
};
}
@@ -220,7 +220,7 @@
factory = new IPartitionWriterFactory() {
@Override
public IFrameWriter createFrameWriter(int receiverIndex) throws HyracksDataException {
- return new PipelinedPartition(ncs.getPartitionManager(), new PartitionId(jobId,
+ return new PipelinedPartition(ctx, ncs.getPartitionManager(), new PartitionId(jobId,
conn.getConnectorId(), senderIndex, receiverIndex), taId);
}
};
diff --git a/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/accessors/UTF8StringBinaryHashFunctionFamily.java b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/accessors/UTF8StringBinaryHashFunctionFamily.java
index e5beadf..8fd0116 100644
--- a/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/accessors/UTF8StringBinaryHashFunctionFamily.java
+++ b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/accessors/UTF8StringBinaryHashFunctionFamily.java
@@ -37,7 +37,7 @@
@Override
public int hash(byte[] bytes, int offset, int length) {
int h = 0;
- int utflen = UTF8StringPointable.getUTFLen(bytes, offset);
+ int utflen = UTF8StringPointable.getUTFLength(bytes, offset);
int sStart = offset + 2;
int c = 0;
diff --git a/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/algorithms/BinarySearchAlgorithm.java b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/algorithms/BinarySearchAlgorithm.java
new file mode 100644
index 0000000..e631428
--- /dev/null
+++ b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/algorithms/BinarySearchAlgorithm.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.algorithms;
+
+import edu.uci.ics.hyracks.data.std.api.IComparable;
+import edu.uci.ics.hyracks.data.std.collections.api.IValueReferenceVector;
+
+/**
+ * Performs Binary Search over a vector of value references.
+ * Assumption: The items in the vector are sorted in ascending order with respect
+ * to the specified key.
+ *
+ * @author vinayakb
+ */
+public class BinarySearchAlgorithm {
+ private int index;
+
+ public BinarySearchAlgorithm() {
+ }
+
+ /**
+ * Perform binary search on the given vector for the specified key. It returns the result of the search
+ * and sets the {@link #index} value that is retrievable by calling {@link #getIndex()} to refer to
+ * the index in the array where the key was found (if it was found), and where it should have been (if
+ * not found).
+ *
+ * @param vector
+ * - Sorted vector of items
+ * @param key
+ * - Search key
+ * @return <code>true</code> if the key is found, <code>false</code> otherwise.
+ */
+ public boolean find(IValueReferenceVector vector, IComparable key) {
+ index = 0;
+ int left = 0;
+ int right = vector.getSize() - 1;
+ while (left <= right) {
+ index = (left + right) / 2;
+ int cmp = key.compareTo(vector.getBytes(index), vector.getStart(index), vector.getLength(index));
+ if (cmp > 0) {
+ left = index + 1;
+ index = left;
+ } else if (cmp < 0) {
+ right = index - 1;
+ index = left;
+ } else {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Get the index as a result of binary search.
+ *
+ * @return the index in the array where the key was found (if it was found), and where it should have been (if
+ * not found).
+ */
+ public int getIndex() {
+ return index;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/AbstractPointable.java b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/AbstractPointable.java
index 2d0d87c..3ee6b0e 100644
--- a/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/AbstractPointable.java
+++ b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/AbstractPointable.java
@@ -29,7 +29,7 @@
}
@Override
- public void set(IPointable pointer) {
+ public void set(IValueReference pointer) {
set(pointer.getByteArray(), pointer.getStartOffset(), pointer.getLength());
}
diff --git a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/IDataOutputProvider.java b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IDataOutputProvider.java
similarity index 92%
rename from hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/IDataOutputProvider.java
rename to hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IDataOutputProvider.java
index fdfe2ec..d64a23a 100644
--- a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/IDataOutputProvider.java
+++ b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IDataOutputProvider.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.dataflow.common.data.accessors;
+package edu.uci.ics.hyracks.data.std.api;
import java.io.DataOutput;
diff --git a/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IMutableValueStorage.java b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IMutableValueStorage.java
new file mode 100644
index 0000000..173a4e6
--- /dev/null
+++ b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IMutableValueStorage.java
@@ -0,0 +1,5 @@
+package edu.uci.ics.hyracks.data.std.api;
+
+public interface IMutableValueStorage extends IValueReference, IDataOutputProvider {
+ public void reset();
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IPointable.java b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IPointable.java
index 17e9b03..d3accba 100644
--- a/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IPointable.java
+++ b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IPointable.java
@@ -14,14 +14,8 @@
*/
package edu.uci.ics.hyracks.data.std.api;
-public interface IPointable {
+public interface IPointable extends IValueReference {
public void set(byte[] bytes, int start, int length);
- public void set(IPointable pointer);
-
- public byte[] getByteArray();
-
- public int getStartOffset();
-
- public int getLength();
+ public void set(IValueReference pointer);
}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/IValueReference.java b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IValueReference.java
similarity index 84%
rename from hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/IValueReference.java
rename to hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IValueReference.java
index 3c005de..ebe8e1f 100644
--- a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/IValueReference.java
+++ b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IValueReference.java
@@ -12,12 +12,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.dataflow.common.data.accessors;
+package edu.uci.ics.hyracks.data.std.api;
public interface IValueReference {
- public byte[] getBytes();
+ public byte[] getByteArray();
- public int getStartIndex();
+ public int getStartOffset();
public int getLength();
}
\ No newline at end of file
diff --git a/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/collections/api/IValueReferenceVector.java b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/collections/api/IValueReferenceVector.java
new file mode 100644
index 0000000..090f9d4
--- /dev/null
+++ b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/collections/api/IValueReferenceVector.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.collections.api;
+
+/**
+ * Represents an immutable vector of ValueReferences. Users of this interface make the assumption
+ * that it provides random access to the items in it. In other words, each of the getXXX(index) calls
+ * have O(1) complexity.
+ *
+ * @author vinayakb
+ */
+public interface IValueReferenceVector {
+ /**
+ * Number of items in this vector.
+ *
+ * @return - the number of items in this vector.
+ */
+ public int getSize();
+
+ /**
+ * Get the byte array that contains the item referenced by <code>index</code>
+ *
+ * @param index
+ * - Index of the item in the vector
+ * @return Byte Array that contains the item.
+ */
+ public byte[] getBytes(int index);
+
+ /**
+ * Get the start offset of the item referenced by <code>index</code>
+ *
+ * @param index
+ * - Index of the item in the vector
+ * @return Start Offset in the Byte Array of the item
+ */
+ public int getStart(int index);
+
+ /**
+ * Get the length of the item referenced by <code>index</code>
+ *
+ * @param index
+ * - Index of the item in the vector
+ * @return Length in the Byte Array of the item
+ */
+ public int getLength(int index);
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/ShortPointable.java b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/ShortPointable.java
index 57b9a1b..c9022ab 100644
--- a/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/ShortPointable.java
+++ b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/ShortPointable.java
@@ -56,8 +56,8 @@
}
public static void setShort(byte[] bytes, int start, short value) {
- bytes[start] = (byte) ((value >>> 24) & 0xFF);
- bytes[start + 1] = (byte) ((value >>> 16) & 0xFF);
+ bytes[start] = (byte) ((value >>> 8) & 0xFF);
+ bytes[start + 1] = (byte) ((value >>> 0) & 0xFF);
}
public short getShort() {
diff --git a/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/UTF8StringPointable.java b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/UTF8StringPointable.java
index 21caf77..f6d6093 100644
--- a/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/UTF8StringPointable.java
+++ b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/UTF8StringPointable.java
@@ -50,6 +50,18 @@
}
};
+ /**
+ * Returns the character at the given byte offset. The caller is responsible for making sure that
+ * the provided offset is within bounds and points to the beginning of a valid UTF8 character.
+ *
+ * @param offset
+ * - Byte offset
+ * @return Character at the given offset.
+ */
+ public char charAt(int offset) {
+ return charAt(bytes, start + offset);
+ }
+
public static char charAt(byte[] b, int s) {
int c = b[s] & 0xff;
switch (c >> 4) {
@@ -75,6 +87,10 @@
}
}
+ public int charSize(int offset) {
+ return charSize(bytes, start + offset);
+ }
+
public static int charSize(byte[] b, int s) {
int c = b[s] & 0xff;
switch (c >> 4) {
@@ -108,9 +124,18 @@
}
}
- public static int getStrLen(byte[] b, int s) {
+ /**
+ * Gets the length of the string in characters.
+ *
+ * @return length of string in characters
+ */
+ public int getStringLength() {
+ return getStringLength(bytes, start);
+ }
+
+ public static int getStringLength(byte[] b, int s) {
int pos = s + 2;
- int end = pos + getUTFLen(b, s);
+ int end = pos + getUTFLength(b, s);
int charCount = 0;
while (pos < end) {
charCount++;
@@ -119,7 +144,16 @@
return charCount;
}
- public static int getUTFLen(byte[] b, int s) {
+ /**
+ * Gets the length of the UTF-8 encoded string in bytes.
+ *
+ * @return length of UTF-8 encoded string in bytes
+ */
+ public int getUTFLength() {
+ return getUTFLength(bytes, start);
+ }
+
+ public static int getUTFLength(byte[] b, int s) {
return ((b[s] & 0xff) << 8) + ((b[s + 1] & 0xff) << 0);
}
@@ -130,8 +164,8 @@
@Override
public int compareTo(byte[] bytes, int start, int length) {
- int utflen1 = getUTFLen(this.bytes, this.start);
- int utflen2 = getUTFLen(bytes, start);
+ int utflen1 = getUTFLength(this.bytes, this.start);
+ int utflen2 = getUTFLength(bytes, start);
int c1 = 0;
int c2 = 0;
@@ -155,7 +189,7 @@
@Override
public int hash() {
int h = 0;
- int utflen = getUTFLen(bytes, start);
+ int utflen = getUTFLength(bytes, start);
int sStart = start + 2;
int c = 0;
@@ -166,4 +200,20 @@
}
return h;
}
+
+ public static void toString(StringBuilder buffer, byte[] bytes, int start) {
+ int utfLen = getUTFLength(bytes, start);
+ int offset = 2;
+ while (utfLen > 0) {
+ char c = charAt(bytes, start + offset);
+ buffer.append(c);
+ int cLen = UTF8StringPointable.getModifiedUTF8Len(c);
+ offset += cLen;
+ utfLen -= cLen;
+ }
+ }
+
+ public void toString(StringBuilder buffer) {
+ toString(buffer, bytes, start);
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/UTF8StringWriter.java b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/UTF8StringWriter.java
new file mode 100644
index 0000000..4eb489f
--- /dev/null
+++ b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/UTF8StringWriter.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.primitive;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.UTFDataFormatException;
+
+public class UTF8StringWriter {
+ private byte[] tempBytes;
+
+ public void writeUTF8String(CharSequence str, DataOutput out) throws IOException {
+ int strlen = str.length();
+ int utflen = 0;
+ int c, count = 0;
+
+ for (int i = 0; i < strlen; i++) {
+ c = str.charAt(i);
+ if ((c >= 0x0001) && (c <= 0x007F)) {
+ utflen++;
+ } else if (c > 0x07FF) {
+ utflen += 3;
+ } else {
+ utflen += 2;
+ }
+ }
+
+ if (utflen > 65535) {
+ throw new UTFDataFormatException("encoded string too long: " + utflen + " bytes");
+ }
+
+ if (tempBytes == null || tempBytes.length < utflen + 2) {
+ tempBytes = new byte[utflen + 2];
+ }
+
+ tempBytes[count++] = (byte) ((utflen >>> 8) & 0xFF);
+ tempBytes[count++] = (byte) ((utflen >>> 0) & 0xFF);
+
+ int i = 0;
+ for (i = 0; i < strlen; i++) {
+ c = str.charAt(i);
+ if (!((c >= 0x0001) && (c <= 0x007F))) {
+ break;
+ }
+ tempBytes[count++] = (byte) c;
+ }
+
+ for (; i < strlen; i++) {
+ c = str.charAt(i);
+ if ((c >= 0x0001) && (c <= 0x007F)) {
+ tempBytes[count++] = (byte) c;
+ } else if (c > 0x07FF) {
+ tempBytes[count++] = (byte) (0xE0 | ((c >> 12) & 0x0F));
+ tempBytes[count++] = (byte) (0x80 | ((c >> 6) & 0x3F));
+ tempBytes[count++] = (byte) (0x80 | ((c >> 0) & 0x3F));
+ } else {
+ tempBytes[count++] = (byte) (0xC0 | ((c >> 6) & 0x1F));
+ tempBytes[count++] = (byte) (0x80 | ((c >> 0) & 0x3F));
+ }
+ }
+ out.write(tempBytes, 0, utflen + 2);
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/util/ArrayBackedValueStorage.java b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/util/ArrayBackedValueStorage.java
new file mode 100644
index 0000000..e8dc9b4
--- /dev/null
+++ b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/util/ArrayBackedValueStorage.java
@@ -0,0 +1,56 @@
+package edu.uci.ics.hyracks.data.std.util;
+
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.data.std.api.IMutableValueStorage;
+import edu.uci.ics.hyracks.data.std.api.IValueReference;
+
+public class ArrayBackedValueStorage implements IMutableValueStorage {
+ private final ByteArrayAccessibleOutputStream baaos;
+ private final DataOutputStream dos;
+
+ public ArrayBackedValueStorage() {
+ baaos = new ByteArrayAccessibleOutputStream();
+ dos = new DataOutputStream(baaos);
+ }
+
+ @Override
+ public void reset() {
+ baaos.reset();
+ }
+
+ @Override
+ public DataOutput getDataOutput() {
+ return dos;
+ }
+
+ @Override
+ public byte[] getByteArray() {
+ return baaos.getByteArray();
+ }
+
+ @Override
+ public int getStartOffset() {
+ return 0;
+ }
+
+ @Override
+ public int getLength() {
+ return baaos.size();
+ }
+
+ public void append(IValueReference value) {
+ try {
+ dos.write(value.getByteArray(), value.getStartOffset(), value.getLength());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void assign(IValueReference value) {
+ reset();
+ append(value);
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/ByteArrayAccessibleOutputStream.java b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/util/ByteArrayAccessibleOutputStream.java
similarity index 96%
rename from hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/ByteArrayAccessibleOutputStream.java
rename to hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/util/ByteArrayAccessibleOutputStream.java
index 31bf554..8508287 100644
--- a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/ByteArrayAccessibleOutputStream.java
+++ b/hyracks/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/util/ByteArrayAccessibleOutputStream.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.dataflow.common.comm.io;
+package edu.uci.ics.hyracks.data.std.util;
import java.io.ByteArrayOutputStream;
import java.util.logging.Level;
diff --git a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/ArrayTupleBuilder.java b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/ArrayTupleBuilder.java
index 0e9e3c0..989bc6b 100644
--- a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/ArrayTupleBuilder.java
+++ b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/ArrayTupleBuilder.java
@@ -21,7 +21,8 @@
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
+import edu.uci.ics.hyracks.data.std.util.ByteArrayAccessibleOutputStream;
/**
* Array backed tuple builder.
diff --git a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameDeserializingDataWriter.java b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameDeserializingDataWriter.java
index 2ec4b47..b095332 100644
--- a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameDeserializingDataWriter.java
+++ b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameDeserializingDataWriter.java
@@ -17,7 +17,7 @@
import java.nio.ByteBuffer;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.IOpenableDataWriter;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
@@ -26,7 +26,7 @@
private final IOpenableDataWriter<Object[]> writer;
private final FrameDeserializer frameDeserializer;
- public FrameDeserializingDataWriter(IHyracksRootContext ctx, IOpenableDataWriter<Object[]> writer,
+ public FrameDeserializingDataWriter(IHyracksTaskContext ctx, IOpenableDataWriter<Object[]> writer,
RecordDescriptor recordDescriptor) {
this.writer = writer;
this.frameDeserializer = new FrameDeserializer(ctx.getFrameSize(), recordDescriptor);
diff --git a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/ArrayBackedValueStorage.java b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/ArrayBackedValueStorage.java
deleted file mode 100644
index 44a7a73..0000000
--- a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/ArrayBackedValueStorage.java
+++ /dev/null
@@ -1,40 +0,0 @@
-package edu.uci.ics.hyracks.dataflow.common.data.accessors;
-
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ByteArrayAccessibleOutputStream;
-
-public class ArrayBackedValueStorage implements IValueReference, IDataOutputProvider {
- private final ByteArrayAccessibleOutputStream baaos;
- private final DataOutputStream dos;
-
- public ArrayBackedValueStorage() {
- baaos = new ByteArrayAccessibleOutputStream();
- dos = new DataOutputStream(baaos);
- }
-
- public void reset() {
- baaos.reset();
- }
-
- @Override
- public DataOutput getDataOutput() {
- return dos;
- }
-
- @Override
- public byte[] getBytes() {
- return baaos.getByteArray();
- }
-
- @Override
- public int getStartIndex() {
- return 0;
- }
-
- @Override
- public int getLength() {
- return baaos.size();
- }
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/FrameTupleFieldValueReference.java b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/FrameTupleFieldValueReference.java
index 3cddbac..375be2d 100644
--- a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/FrameTupleFieldValueReference.java
+++ b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/FrameTupleFieldValueReference.java
@@ -15,6 +15,7 @@
package edu.uci.ics.hyracks.dataflow.common.data.accessors;
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.data.std.api.IValueReference;
public class FrameTupleFieldValueReference implements IValueReference {
private IFrameTupleAccessor fta;
@@ -31,12 +32,12 @@
}
@Override
- public byte[] getBytes() {
+ public byte[] getByteArray() {
return fta.getBuffer().array();
}
@Override
- public int getStartIndex() {
+ public int getStartOffset() {
return fta.getFieldStartOffset(tupleIndex, fieldIndex);
}
diff --git a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/normalizers/UTF8StringNormalizedKeyComputerFactory.java b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/normalizers/UTF8StringNormalizedKeyComputerFactory.java
index 60fd395..7d7ebac 100644
--- a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/normalizers/UTF8StringNormalizedKeyComputerFactory.java
+++ b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/normalizers/UTF8StringNormalizedKeyComputerFactory.java
@@ -26,7 +26,7 @@
return new INormalizedKeyComputer() {
@Override
public int normalize(byte[] bytes, int start, int length) {
- int len = UTF8StringPointable.getUTFLen(bytes, start);
+ int len = UTF8StringPointable.getUTFLength(bytes, start);
int nk = 0;
int offset = start + 2;
for (int i = 0; i < 2; ++i) {
diff --git a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerFactory.java b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerFactory.java
index 7769539..f6d4d51 100644
--- a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerFactory.java
+++ b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerFactory.java
@@ -55,7 +55,7 @@
h = h * 31 + fh;
}
if (h < 0) {
- h = -h;
+ h = -(h + 1);
}
return h % nParts;
}
diff --git a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/io/RunFileReader.java b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/io/RunFileReader.java
index d0148f3..a706692 100644
--- a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/io/RunFileReader.java
+++ b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/io/RunFileReader.java
@@ -4,8 +4,8 @@
import edu.uci.ics.hyracks.api.comm.IFrameReader;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileHandle;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IFileHandle;
import edu.uci.ics.hyracks.api.io.IIOManager;
public class RunFileReader implements IFrameReader {
@@ -13,7 +13,7 @@
private final IIOManager ioManager;
private final long size;
- private FileHandle handle;
+ private IFileHandle handle;
private long readPtr;
public RunFileReader(FileReference file, IIOManager ioManager, long size) {
@@ -42,7 +42,7 @@
public void close() throws HyracksDataException {
ioManager.close(handle);
}
-
+
public long getFileSize() {
return size;
}
diff --git a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/io/RunFileWriter.java b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/io/RunFileWriter.java
index 6c5a957..37a1a2c 100644
--- a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/io/RunFileWriter.java
+++ b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/io/RunFileWriter.java
@@ -18,8 +18,8 @@
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileHandle;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IFileHandle;
import edu.uci.ics.hyracks.api.io.IIOManager;
public class RunFileWriter implements IFrameWriter {
@@ -27,7 +27,7 @@
private final IIOManager ioManager;
private boolean failed;
- private FileHandle handle;
+ private IFileHandle handle;
private long size;
public RunFileWriter(FileReference file, IIOManager ioManager) {
diff --git a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/SerdeUtils.java b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/SerdeUtils.java
index 87d9b35..00575f4 100644
--- a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/SerdeUtils.java
+++ b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/SerdeUtils.java
@@ -35,13 +35,41 @@
@SuppressWarnings("rawtypes")
public class SerdeUtils {
- public static ITypeTraits[] serdesToTypeTraits(ISerializerDeserializer[] serdes, int numSerdes) {
- ITypeTraits[] typeTraits = new ITypeTraits[numSerdes];
- for (int i = 0; i < numSerdes; i++) {
+ public static class PayloadTypeTraits implements ITypeTraits {
+ private static final long serialVersionUID = 1L;
+ final int payloadSize;
+
+ public PayloadTypeTraits(int payloadSize) {
+ this.payloadSize = payloadSize;
+ }
+
+ @Override
+ public boolean isFixedLength() {
+ return true;
+ }
+
+ @Override
+ public int getFixedLength() {
+ return payloadSize;
+ }
+ }
+
+ public static ITypeTraits[] serdesToTypeTraits(ISerializerDeserializer[] serdes) {
+ ITypeTraits[] typeTraits = new ITypeTraits[serdes.length];
+ for (int i = 0; i < serdes.length; i++) {
typeTraits[i] = serdeToTypeTrait(serdes[i]);
}
return typeTraits;
}
+
+ public static ITypeTraits[] serdesToTypeTraits(ISerializerDeserializer[] serdes, int payloadSize) {
+ ITypeTraits[] typeTraits = new ITypeTraits[serdes.length + 1];
+ for (int i = 0; i < serdes.length; i++) {
+ typeTraits[i] = serdeToTypeTrait(serdes[i]);
+ }
+ typeTraits[serdes.length] = new PayloadTypeTraits(payloadSize);
+ return typeTraits;
+ }
public static ITypeTraits serdeToTypeTrait(ISerializerDeserializer serde) {
if (serde instanceof IntegerSerializerDeserializer) {
diff --git a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/TupleUtils.java b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/TupleUtils.java
index 14c4d66..b35dd75 100644
--- a/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/TupleUtils.java
+++ b/hyracks/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/TupleUtils.java
@@ -25,29 +25,32 @@
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-@SuppressWarnings("rawtypes")
-public class TupleUtils {
+@SuppressWarnings("rawtypes")
+public class TupleUtils {
@SuppressWarnings("unchecked")
- public static void createTuple(ArrayTupleBuilder tupleBuilder, ArrayTupleReference tuple, ISerializerDeserializer[] fieldSerdes, final Object... fields) throws HyracksDataException {
+ public static void createTuple(ArrayTupleBuilder tupleBuilder, ArrayTupleReference tuple,
+ ISerializerDeserializer[] fieldSerdes, final Object... fields) throws HyracksDataException {
DataOutput dos = tupleBuilder.getDataOutput();
tupleBuilder.reset();
int numFields = Math.min(tupleBuilder.getFieldEndOffsets().length, fields.length);
- for (int i = 0; i < numFields; i++) {
+ for (int i = 0; i < numFields; i++) {
fieldSerdes[i].serialize(fields[i], dos);
tupleBuilder.addFieldEndOffset();
}
tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
}
- public static ITupleReference createTuple(ISerializerDeserializer[] fieldSerdes, final Object... fields) throws HyracksDataException {
+ public static ITupleReference createTuple(ISerializerDeserializer[] fieldSerdes, final Object... fields)
+ throws HyracksDataException {
ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fields.length);
ArrayTupleReference tuple = new ArrayTupleReference();
createTuple(tupleBuilder, tuple, fieldSerdes, fields);
return tuple;
}
-
+
public static void createIntegerTuple(ArrayTupleBuilder tupleBuilder, ArrayTupleReference tuple,
final int... fields) throws HyracksDataException {
DataOutput dos = tupleBuilder.getDataOutput();
@@ -65,14 +68,31 @@
createIntegerTuple(tupleBuilder, tuple, fields);
return tuple;
}
-
- public static String printTuple(ITupleReference tuple,
- ISerializerDeserializer[] fields) throws HyracksDataException {
+
+ public static void createDoubleTuple(ArrayTupleBuilder tupleBuilder, ArrayTupleReference tuple,
+ final double... fields) throws HyracksDataException {
+ DataOutput dos = tupleBuilder.getDataOutput();
+ tupleBuilder.reset();
+ for (final double i : fields) {
+ DoubleSerializerDeserializer.INSTANCE.serialize(i, dos);
+ tupleBuilder.addFieldEndOffset();
+ }
+ tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
+ }
+
+ public static ITupleReference createDoubleTuple(final double... fields) throws HyracksDataException {
+ ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fields.length);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ createDoubleTuple(tupleBuilder, tuple, fields);
+ return tuple;
+ }
+
+ public static String printTuple(ITupleReference tuple, ISerializerDeserializer[] fields)
+ throws HyracksDataException {
StringBuilder strBuilder = new StringBuilder();
int numPrintFields = Math.min(tuple.getFieldCount(), fields.length);
for (int i = 0; i < numPrintFields; i++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(
- tuple.getFieldData(i), tuple.getFieldStart(i),
+ ByteArrayInputStream inStream = new ByteArrayInputStream(tuple.getFieldData(i), tuple.getFieldStart(i),
tuple.getFieldLength(i));
DataInput dataIn = new DataInputStream(inStream);
Object o = fields[i].deserialize(dataIn);
@@ -80,7 +100,30 @@
if (i != fields.length - 1) {
strBuilder.append(" ");
}
- }
+ }
return strBuilder.toString();
}
+
+ public static Object[] deserializeTuple(ITupleReference tuple, ISerializerDeserializer[] fields)
+ throws HyracksDataException {
+ int numFields = Math.min(tuple.getFieldCount(), fields.length);
+ Object[] objs = new Object[numFields];
+ for (int i = 0; i < numFields; i++) {
+ ByteArrayInputStream inStream = new ByteArrayInputStream(tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i));
+ DataInput dataIn = new DataInputStream(inStream);
+ objs[i] = fields[i].deserialize(dataIn);
+ }
+ return objs;
+ }
+
+ public static ITupleReference copyTuple(ITupleReference tuple) throws HyracksDataException {
+ ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(tuple.getFieldCount());
+ for (int i = 0; i < tuple.getFieldCount(); i++) {
+ tupleBuilder.addField(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
+ }
+ ArrayTupleReference tupleCopy = new ArrayTupleReference();
+ tupleCopy.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
+ return tupleCopy;
+ }
}
diff --git a/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/AbstractHadoopOperatorDescriptor.java b/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/AbstractHadoopOperatorDescriptor.java
index 89831e8..70dfd9e 100644
--- a/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/AbstractHadoopOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/AbstractHadoopOperatorDescriptor.java
@@ -27,7 +27,7 @@
import edu.uci.ics.dcache.client.DCacheClient;
import edu.uci.ics.hyracks.api.dataflow.IDataWriter;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.hadoop.util.DatatypeHelper;
import edu.uci.ics.hyracks.dataflow.hadoop.util.IHadoopClassFactory;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
@@ -63,7 +63,7 @@
private final Map<String, String> jobConfMap;
private IHadoopClassFactory hadoopClassFactory;
- public AbstractHadoopOperatorDescriptor(JobSpecification spec, int inputArity, RecordDescriptor recordDescriptor,
+ public AbstractHadoopOperatorDescriptor(IOperatorDescriptorRegistry spec, int inputArity, RecordDescriptor recordDescriptor,
JobConf jobConf, IHadoopClassFactory hadoopOperatorFactory) {
super(spec, inputArity, 1);
jobConfMap = DatatypeHelper.jobConf2Map(jobConf);
diff --git a/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopMapperOperatorDescriptor.java b/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopMapperOperatorDescriptor.java
index 0ea56c4..34f3e9d 100644
--- a/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopMapperOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopMapperOperatorDescriptor.java
@@ -40,7 +40,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.common.comm.io.SerializingDataWriter;
import edu.uci.ics.hyracks.dataflow.hadoop.util.DatatypeHelper;
import edu.uci.ics.hyracks.dataflow.hadoop.util.IHadoopClassFactory;
@@ -230,9 +230,7 @@
data[1] = value;
writer.writeData(data);
}
- };
- ;
- ;
+ };;;
OutputCommitter outputCommitter = new org.apache.hadoop.mapreduce.lib.output.NullOutputFormat()
.getOutputCommitter(new TaskAttemptContext(conf, new TaskAttemptID()));
@@ -254,9 +252,7 @@
public Counter getCounter(Enum<?> arg0) {
return null;
}
- };
- ;
- ;
+ };;;
context = new org.apache.hadoop.mapreduce.Mapper().new Context(conf, new TaskAttemptID(),
newReader, recordWriter, outputCommitter, statusReporter,
(org.apache.hadoop.mapreduce.InputSplit) inputSplit);
@@ -308,12 +304,12 @@
inputSplitsProxy = new InputSplitsProxy(jobConf, splits);
}
- public HadoopMapperOperatorDescriptor(JobSpecification spec, JobConf jobConf, IHadoopClassFactory hadoopClassFactory)
- throws IOException {
+ public HadoopMapperOperatorDescriptor(IOperatorDescriptorRegistry spec, JobConf jobConf,
+ IHadoopClassFactory hadoopClassFactory) throws IOException {
super(spec, 1, getRecordDescriptor(jobConf, hadoopClassFactory), jobConf, hadoopClassFactory);
}
- public HadoopMapperOperatorDescriptor(JobSpecification spec, JobConf jobConf, Object[] splits,
+ public HadoopMapperOperatorDescriptor(IOperatorDescriptorRegistry spec, JobConf jobConf, Object[] splits,
IHadoopClassFactory hadoopClassFactory) throws IOException {
super(spec, 0, getRecordDescriptor(jobConf, hadoopClassFactory), jobConf, hadoopClassFactory);
initializeSplitInfo(splits);
@@ -412,7 +408,7 @@
return createSelfReadingMapper(ctx, recordDescriptor, partition);
} else {
return new DeserializedOperatorNodePushable(ctx, new MapperOperator(partition),
- recordDescProvider.getInputRecordDescriptor(this.odId, 0));
+ recordDescProvider.getInputRecordDescriptor(this.activityNodeId, 0));
}
} catch (Exception e) {
throw new HyracksDataException(e);
diff --git a/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopReducerOperatorDescriptor.java b/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopReducerOperatorDescriptor.java
index 73a8c16..86a4c3c 100644
--- a/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopReducerOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopReducerOperatorDescriptor.java
@@ -43,7 +43,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.hadoop.data.KeyComparatorFactory;
import edu.uci.ics.hyracks.dataflow.hadoop.data.RawComparingComparatorFactory;
import edu.uci.ics.hyracks.dataflow.hadoop.util.DatatypeHelper;
@@ -311,8 +311,8 @@
private IComparatorFactory comparatorFactory;
private boolean useAsCombiner = false;
- public HadoopReducerOperatorDescriptor(JobSpecification spec, JobConf conf, IComparatorFactory comparatorFactory,
- IHadoopClassFactory classFactory, boolean useAsCombiner) {
+ public HadoopReducerOperatorDescriptor(IOperatorDescriptorRegistry spec, JobConf conf,
+ IComparatorFactory comparatorFactory, IHadoopClassFactory classFactory, boolean useAsCombiner) {
super(spec, 1, getRecordDescriptor(conf, classFactory), conf, classFactory);
this.comparatorFactory = comparatorFactory;
this.useAsCombiner = useAsCombiner;
@@ -371,7 +371,7 @@
IOpenableDataWriterOperator op = new DeserializedPreclusteredGroupOperator(new int[] { 0 },
new IComparator[] { comparatorFactory.createComparator() }, new ReducerAggregator(createReducer()));
return new DeserializedOperatorNodePushable(ctx, op, recordDescProvider.getInputRecordDescriptor(
- getOperatorId(), 0));
+ getActivityId(), 0));
} catch (Exception e) {
throw new RuntimeException(e);
}
diff --git a/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopWriteOperatorDescriptor.java b/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopWriteOperatorDescriptor.java
index 2154f5a..7ce3b00 100644
--- a/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopWriteOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopWriteOperatorDescriptor.java
@@ -34,7 +34,7 @@
import org.apache.hadoop.util.ReflectionUtils;
import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.hadoop.util.DatatypeHelper;
import edu.uci.ics.hyracks.dataflow.std.file.AbstractFileWriteOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
@@ -226,7 +226,7 @@
}
}
- public HadoopWriteOperatorDescriptor(JobSpecification jobSpec, JobConf jobConf, int numMapTasks) throws Exception {
+ public HadoopWriteOperatorDescriptor(IOperatorDescriptorRegistry jobSpec, JobConf jobConf, int numMapTasks) throws Exception {
super(jobSpec, getOutputSplits(jobConf, numMapTasks));
this.jobConfMap = DatatypeHelper.jobConf2Map(jobConf);
checkIfCanWriteToHDFS(super.splits);
diff --git a/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/HashPartitioningShuffleConnectorDescriptor.java b/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/HashPartitioningShuffleConnectorDescriptor.java
index 7c6bf86..5e00e96 100644
--- a/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/HashPartitioningShuffleConnectorDescriptor.java
+++ b/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/HashPartitioningShuffleConnectorDescriptor.java
@@ -26,7 +26,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractMToNConnectorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicChannelReader;
import edu.uci.ics.hyracks.dataflow.std.collectors.PartitionCollector;
@@ -37,7 +37,7 @@
private final MarshalledWritable<Configuration> mConfig;
- public HashPartitioningShuffleConnectorDescriptor(JobSpecification spec, MarshalledWritable<Configuration> mConfig) {
+ public HashPartitioningShuffleConnectorDescriptor(IConnectorDescriptorRegistry spec, MarshalledWritable<Configuration> mConfig) {
super(spec);
this.mConfig = mConfig;
}
diff --git a/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/MapperOperatorDescriptor.java b/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/MapperOperatorDescriptor.java
index 4a61296..4b1144a 100644
--- a/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/MapperOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/MapperOperatorDescriptor.java
@@ -36,7 +36,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
@@ -53,7 +53,7 @@
private final MarshalledWritable<Configuration> config;
private final IInputSplitProviderFactory factory;
- public MapperOperatorDescriptor(JobSpecification spec, int jobId, MarshalledWritable<Configuration> config,
+ public MapperOperatorDescriptor(IOperatorDescriptorRegistry spec, int jobId, MarshalledWritable<Configuration> config,
IInputSplitProviderFactory factory) throws HyracksDataException {
super(spec, 0, 1);
this.jobId = jobId;
diff --git a/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/ReducerOperatorDescriptor.java b/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/ReducerOperatorDescriptor.java
index bbb0d74..d6cab23 100644
--- a/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/ReducerOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/ReducerOperatorDescriptor.java
@@ -29,7 +29,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
@@ -41,7 +41,7 @@
private MarshalledWritable<Configuration> mConfig;
- public ReducerOperatorDescriptor(JobSpecification spec, int jobId, MarshalledWritable<Configuration> mConfig) {
+ public ReducerOperatorDescriptor(IOperatorDescriptorRegistry spec, int jobId, MarshalledWritable<Configuration> mConfig) {
super(spec, 1, 0);
this.jobId = jobId;
this.mConfig = mConfig;
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java
index 7c50da9..ef479e1 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java
@@ -21,8 +21,8 @@
import edu.uci.ics.hyracks.api.constraints.IConstraintAcceptor;
import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.job.JobActivityGraph;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.ActivityCluster;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
public abstract class AbstractConnectorDescriptor implements IConnectorDescriptor {
private static final long serialVersionUID = 1L;
@@ -30,9 +30,8 @@
protected String displayName;
- public AbstractConnectorDescriptor(JobSpecification spec) {
- this.id = spec.createConnectorDescriptor();
- spec.getConnectorMap().put(id, this);
+ public AbstractConnectorDescriptor(IConnectorDescriptorRegistry spec) {
+ this.id = spec.createConnectorDescriptor(this);
displayName = getClass().getName() + "[" + id + "]";
}
@@ -60,7 +59,7 @@
}
@Override
- public void contributeSchedulingConstraints(IConstraintAcceptor constraintAcceptor, JobActivityGraph plan,
+ public void contributeSchedulingConstraints(IConstraintAcceptor constraintAcceptor, ActivityCluster ac,
ICCApplicationContext appCtx) {
// do nothing
}
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractMToNConnectorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractMToNConnectorDescriptor.java
index ae1ead5..8e9f131 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractMToNConnectorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractMToNConnectorDescriptor.java
@@ -16,12 +16,12 @@
import java.util.BitSet;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
public abstract class AbstractMToNConnectorDescriptor extends AbstractConnectorDescriptor {
private static final long serialVersionUID = 1L;
- public AbstractMToNConnectorDescriptor(JobSpecification spec) {
+ public AbstractMToNConnectorDescriptor(IConnectorDescriptorRegistry spec) {
super(spec);
}
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java
index 983851e..153abd3 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java
@@ -22,8 +22,8 @@
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.api.job.JobActivityGraph;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
public abstract class AbstractOperatorDescriptor implements IOperatorDescriptor {
private static final long serialVersionUID = 1L;
@@ -40,12 +40,11 @@
protected String displayName;
- public AbstractOperatorDescriptor(JobSpecification spec, int inputArity, int outputArity) {
- odId = spec.createOperatorDescriptorId();
+ public AbstractOperatorDescriptor(IOperatorDescriptorRegistry spec, int inputArity, int outputArity) {
+ odId = spec.createOperatorDescriptorId(this);
this.inputArity = inputArity;
this.outputArity = outputArity;
recordDescriptors = new RecordDescriptor[outputArity];
- spec.getOperatorMap().put(getOperatorId(), this);
displayName = getClass().getName() + "[" + odId + "]";
}
@@ -78,8 +77,7 @@
}
@Override
- public void contributeSchedulingConstraints(IConstraintAcceptor constraintAcceptor, JobActivityGraph plan,
- ICCApplicationContext appCtx) {
+ public void contributeSchedulingConstraints(IConstraintAcceptor constraintAcceptor, ICCApplicationContext appCtx) {
// do nothing
}
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractSingleActivityOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractSingleActivityOperatorDescriptor.java
index c4c3326..9048670 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractSingleActivityOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractSingleActivityOperatorDescriptor.java
@@ -17,14 +17,14 @@
import edu.uci.ics.hyracks.api.dataflow.ActivityId;
import edu.uci.ics.hyracks.api.dataflow.IActivity;
import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
public abstract class AbstractSingleActivityOperatorDescriptor extends AbstractOperatorDescriptor implements IActivity {
private static final long serialVersionUID = 1L;
protected final ActivityId activityNodeId;
- public AbstractSingleActivityOperatorDescriptor(JobSpecification spec, int inputArity, int outputArity) {
+ public AbstractSingleActivityOperatorDescriptor(IOperatorDescriptorRegistry spec, int inputArity, int outputArity) {
super(spec, inputArity, outputArity);
activityNodeId = new ActivityId(odId, 0);
}
@@ -36,7 +36,7 @@
@Override
public final void contributeActivities(IActivityGraphBuilder builder) {
- builder.addActivity(this);
+ builder.addActivity(this, this);
for (int i = 0; i < getInputArity(); ++i) {
builder.addSourceEdge(i, this, i);
}
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractStateObject.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractStateObject.java
new file mode 100644
index 0000000..753fb98
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractStateObject.java
@@ -0,0 +1,47 @@
+package edu.uci.ics.hyracks.dataflow.std.base;
+
+import edu.uci.ics.hyracks.api.dataflow.state.IStateObject;
+import edu.uci.ics.hyracks.api.job.JobId;
+
+public abstract class AbstractStateObject implements IStateObject {
+ protected JobId jobId;
+
+ protected Object id;
+
+ protected long memoryOccupancy;
+
+ protected AbstractStateObject() {
+ }
+
+ protected AbstractStateObject(JobId jobId, Object id) {
+ this.jobId = jobId;
+ this.id = id;
+ }
+
+ @Override
+ public final JobId getJobId() {
+ return jobId;
+ }
+
+ public final void setJobId(JobId jobId) {
+ this.jobId = jobId;
+ }
+
+ @Override
+ public final Object getId() {
+ return id;
+ }
+
+ public final void setId(Object id) {
+ this.id = id;
+ }
+
+ @Override
+ public final long getMemoryOccupancy() {
+ return memoryOccupancy;
+ }
+
+ public void setMemoryOccupancy(long memoryOccupancy) {
+ this.memoryOccupancy = memoryOccupancy;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractTaskState.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractTaskState.java
deleted file mode 100644
index ba4dfd1..0000000
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractTaskState.java
+++ /dev/null
@@ -1,48 +0,0 @@
-package edu.uci.ics.hyracks.dataflow.std.base;
-
-import edu.uci.ics.hyracks.api.dataflow.TaskId;
-import edu.uci.ics.hyracks.api.dataflow.state.ITaskState;
-import edu.uci.ics.hyracks.api.job.JobId;
-
-public abstract class AbstractTaskState implements ITaskState {
- protected JobId jobId;
-
- protected TaskId taId;
-
- protected long memoryOccupancy;
-
- protected AbstractTaskState() {
- }
-
- protected AbstractTaskState(JobId jobId, TaskId taId) {
- this.jobId = jobId;
- this.taId = taId;
- }
-
- @Override
- public final JobId getJobId() {
- return jobId;
- }
-
- public final void setJobId(JobId jobId) {
- this.jobId = jobId;
- }
-
- @Override
- public final TaskId getTaskId() {
- return taId;
- }
-
- public final void setTaskId(TaskId tId) {
- this.taId = tId;
- }
-
- @Override
- public final long getMemoryOccupancy() {
- return memoryOccupancy;
- }
-
- public void setMemoryOccupancy(long memoryOccupancy) {
- this.memoryOccupancy = memoryOccupancy;
- }
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/PartitionCollector.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/PartitionCollector.java
index da5b0bc..23df4bb 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/PartitionCollector.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/PartitionCollector.java
@@ -51,7 +51,7 @@
PartitionId pid = pc.getPartitionId();
IInputChannel channel = pc.getInputChannel();
pa.addPartition(pid, channel);
- channel.open();
+ channel.open(ctx);
}
}
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/LocalityAwareMToNPartitioningConnectorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/LocalityAwareMToNPartitioningConnectorDescriptor.java
index 0e1e413..0349633 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/LocalityAwareMToNPartitioningConnectorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/LocalityAwareMToNPartitioningConnectorDescriptor.java
@@ -23,7 +23,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractMToNConnectorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicChannelReader;
import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicFrameReader;
@@ -37,7 +37,7 @@
private ITuplePartitionComputerFactory tpcf;
- public LocalityAwareMToNPartitioningConnectorDescriptor(JobSpecification spec, ITuplePartitionComputerFactory tpcf,
+ public LocalityAwareMToNPartitioningConnectorDescriptor(IConnectorDescriptorRegistry spec, ITuplePartitionComputerFactory tpcf,
ILocalityMap localityMap) {
super(spec);
this.localityMap = localityMap;
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNPartitioningConnectorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNPartitioningConnectorDescriptor.java
index 291e149..451e208 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNPartitioningConnectorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNPartitioningConnectorDescriptor.java
@@ -23,7 +23,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractMToNConnectorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicChannelReader;
import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicFrameReader;
@@ -33,7 +33,7 @@
private static final long serialVersionUID = 1L;
private ITuplePartitionComputerFactory tpcf;
- public MToNPartitioningConnectorDescriptor(JobSpecification spec, ITuplePartitionComputerFactory tpcf) {
+ public MToNPartitioningConnectorDescriptor(IConnectorDescriptorRegistry spec, ITuplePartitionComputerFactory tpcf) {
super(spec);
this.tpcf = tpcf;
}
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNPartitioningMergingConnectorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNPartitioningMergingConnectorDescriptor.java
index 2576423..0d80161 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNPartitioningMergingConnectorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNPartitioningMergingConnectorDescriptor.java
@@ -26,7 +26,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractMToNConnectorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.collectors.IPartitionBatchManager;
import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicPartitionBatchManager;
@@ -41,12 +41,12 @@
private final IBinaryComparatorFactory[] comparatorFactories;
private final boolean stable;
- public MToNPartitioningMergingConnectorDescriptor(JobSpecification spec, ITuplePartitionComputerFactory tpcf,
+ public MToNPartitioningMergingConnectorDescriptor(IConnectorDescriptorRegistry spec, ITuplePartitionComputerFactory tpcf,
int[] sortFields, IBinaryComparatorFactory[] comparatorFactories) {
this(spec, tpcf, sortFields, comparatorFactories, false);
}
- public MToNPartitioningMergingConnectorDescriptor(JobSpecification spec, ITuplePartitionComputerFactory tpcf,
+ public MToNPartitioningMergingConnectorDescriptor(IConnectorDescriptorRegistry spec, ITuplePartitionComputerFactory tpcf,
int[] sortFields, IBinaryComparatorFactory[] comparatorFactories, boolean stable) {
super(spec);
this.tpcf = tpcf;
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNReplicatingConnectorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNReplicatingConnectorDescriptor.java
index fd67f77..2d04938 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNReplicatingConnectorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNReplicatingConnectorDescriptor.java
@@ -23,14 +23,14 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractMToNConnectorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicChannelReader;
import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicFrameReader;
import edu.uci.ics.hyracks.dataflow.std.collectors.PartitionCollector;
public class MToNReplicatingConnectorDescriptor extends AbstractMToNConnectorDescriptor {
- public MToNReplicatingConnectorDescriptor(JobSpecification spec) {
+ public MToNReplicatingConnectorDescriptor(IConnectorDescriptorRegistry spec) {
super(spec);
}
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/OneToOneConnectorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/OneToOneConnectorDescriptor.java
index e8d9751..7d007a4 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/OneToOneConnectorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/OneToOneConnectorDescriptor.java
@@ -24,11 +24,11 @@
import edu.uci.ics.hyracks.api.constraints.IConstraintAcceptor;
import edu.uci.ics.hyracks.api.constraints.expressions.PartitionCountExpression;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobActivityGraph;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.ActivityCluster;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractConnectorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicChannelReader;
import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicFrameReader;
@@ -37,7 +37,7 @@
public class OneToOneConnectorDescriptor extends AbstractConnectorDescriptor {
private static final long serialVersionUID = 1L;
- public OneToOneConnectorDescriptor(JobSpecification spec) {
+ public OneToOneConnectorDescriptor(IConnectorDescriptorRegistry spec) {
super(spec);
}
@@ -60,14 +60,13 @@
}
@Override
- public void contributeSchedulingConstraints(IConstraintAcceptor constraintAcceptor, JobActivityGraph plan,
+ public void contributeSchedulingConstraints(IConstraintAcceptor constraintAcceptor, ActivityCluster ac,
ICCApplicationContext appCtx) {
- JobSpecification jobSpec = plan.getJobSpecification();
- IOperatorDescriptor consumer = jobSpec.getConsumer(this);
- IOperatorDescriptor producer = jobSpec.getProducer(this);
+ OperatorDescriptorId consumer = ac.getConsumerActivity(getConnectorId()).getOperatorDescriptorId();
+ OperatorDescriptorId producer = ac.getProducerActivity(getConnectorId()).getOperatorDescriptorId();
- constraintAcceptor.addConstraint(new Constraint(new PartitionCountExpression(consumer.getOperatorId()),
- new PartitionCountExpression(producer.getOperatorId())));
+ constraintAcceptor.addConstraint(new Constraint(new PartitionCountExpression(consumer),
+ new PartitionCountExpression(producer)));
}
@Override
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java
index 8544891..1cd898e 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java
@@ -22,7 +22,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.IOpenableDataWriterOperator;
import edu.uci.ics.hyracks.dataflow.std.util.DeserializedOperatorNodePushable;
@@ -32,7 +32,7 @@
protected FileSplit[] splits;
- public AbstractDeserializedFileScanOperatorDescriptor(JobSpecification spec, FileSplit[] splits,
+ public AbstractDeserializedFileScanOperatorDescriptor(IOperatorDescriptorRegistry spec, FileSplit[] splits,
RecordDescriptor recordDescriptor) {
super(spec, 0, 1);
recordDescriptors[0] = recordDescriptor;
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java
index 1744b99..e05d80d 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java
@@ -19,7 +19,7 @@
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.IOpenableDataWriterOperator;
import edu.uci.ics.hyracks.dataflow.std.util.DeserializedOperatorNodePushable;
@@ -80,7 +80,7 @@
this.splits = splits;
}
- public AbstractFileWriteOperatorDescriptor(JobSpecification spec, FileSplit[] splits) {
+ public AbstractFileWriteOperatorDescriptor(IOperatorDescriptorRegistry spec, FileSplit[] splits) {
super(spec, 1, 0);
this.splits = splits;
}
@@ -91,6 +91,6 @@
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
return new DeserializedOperatorNodePushable(ctx, new FileWriteOperator(partition),
- recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0));
+ recordDescProvider.getInputRecordDescriptor(getActivityId(), 0));
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java
index 76cb0ae..d78593b 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java
@@ -24,7 +24,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
@@ -35,7 +35,7 @@
private final ITupleParserFactory tupleParserFactory;
- public FileScanOperatorDescriptor(JobSpecification spec, IFileSplitProvider fileSplitProvider,
+ public FileScanOperatorDescriptor(IOperatorDescriptorRegistry spec, IFileSplitProvider fileSplitProvider,
ITupleParserFactory tupleParserFactory, RecordDescriptor rDesc) {
super(spec, 0, 1);
this.fileSplitProvider = fileSplitProvider;
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java
index 8397d29..7f257ea 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java
@@ -24,7 +24,7 @@
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
@@ -33,7 +33,7 @@
private IFileSplitProvider fileSplitProvider;
- public FrameFileWriterOperatorDescriptor(JobSpecification spec, IFileSplitProvider fileSplitProvider) {
+ public FrameFileWriterOperatorDescriptor(IOperatorDescriptorRegistry spec, IFileSplitProvider fileSplitProvider) {
super(spec, 1, 0);
this.fileSplitProvider = fileSplitProvider;
}
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java
index 0760fa5..d5256ee 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java
@@ -18,7 +18,7 @@
import java.io.FileOutputStream;
import java.io.OutputStream;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
public class LineFileWriteOperatorDescriptor extends AbstractFileWriteOperatorDescriptor {
private static final long serialVersionUID = 1L;
@@ -39,15 +39,15 @@
private int[] columns;
private char separator;
- public LineFileWriteOperatorDescriptor(JobSpecification spec, FileSplit[] splits) {
+ public LineFileWriteOperatorDescriptor(IOperatorDescriptorRegistry spec, FileSplit[] splits) {
this(spec, splits, null, RecordWriter.COMMA);
}
- public LineFileWriteOperatorDescriptor(JobSpecification spec, FileSplit[] splits, int[] columns) {
+ public LineFileWriteOperatorDescriptor(IOperatorDescriptorRegistry spec, FileSplit[] splits, int[] columns) {
this(spec, splits, columns, RecordWriter.COMMA);
}
- public LineFileWriteOperatorDescriptor(JobSpecification spec, FileSplit[] splits, int[] columns, char separator) {
+ public LineFileWriteOperatorDescriptor(IOperatorDescriptorRegistry spec, FileSplit[] splits, int[] columns, char separator) {
super(spec, splits);
this.columns = columns;
this.separator = separator;
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java
index 6bf29d2..6fb0189 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java
@@ -25,7 +25,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
@@ -50,7 +50,8 @@
* @param inputArity
* @param outputArity
*/
- public PlainFileWriterOperatorDescriptor(JobSpecification spec, IFileSplitProvider fileSplitProvider, String delim) {
+ public PlainFileWriterOperatorDescriptor(IOperatorDescriptorRegistry spec, IFileSplitProvider fileSplitProvider,
+ String delim) {
super(spec, 1, 0);
this.fileSplitProvider = fileSplitProvider;
this.delim = delim;
@@ -74,9 +75,9 @@
final FileSplit[] splits = fileSplitProvider.getFileSplits();
// Frame accessor
final FrameTupleAccessor frameTupleAccessor = new FrameTupleAccessor(ctx.getFrameSize(),
- recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0));
+ recordDescProvider.getInputRecordDescriptor(getActivityId(), 0));
// Record descriptor
- final RecordDescriptor recordDescriptor = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0);
+ final RecordDescriptor recordDescriptor = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);
return new AbstractUnaryInputSinkOperatorNodePushable() {
private BufferedWriter out;
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java
index af345aa..36e07c4 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java
@@ -21,12 +21,12 @@
import java.io.IOException;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
public class RecordFileScanOperatorDescriptor extends AbstractDeserializedFileScanOperatorDescriptor {
private static final long serialVersionUID = 1L;
- public RecordFileScanOperatorDescriptor(JobSpecification spec, FileSplit[] splits, RecordDescriptor recordDescriptor) {
+ public RecordFileScanOperatorDescriptor(IOperatorDescriptorRegistry spec, FileSplit[] splits, RecordDescriptor recordDescriptor) {
super(spec, splits, recordDescriptor);
}
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ExternalGroupOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ExternalGroupOperatorDescriptor.java
deleted file mode 100644
index 58f9ffa..0000000
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ExternalGroupOperatorDescriptor.java
+++ /dev/null
@@ -1,675 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.group;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.LinkedList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.ActivityId;
-import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
-import edu.uci.ics.hyracks.api.dataflow.TaskId;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
-import edu.uci.ics.hyracks.dataflow.common.io.RunFileReader;
-import edu.uci.ics.hyracks.dataflow.common.io.RunFileWriter;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractTaskState;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
-import edu.uci.ics.hyracks.dataflow.std.util.ReferenceEntry;
-import edu.uci.ics.hyracks.dataflow.std.util.ReferencedPriorityQueue;
-
-/**
- *
- */
-public class ExternalGroupOperatorDescriptor extends AbstractOperatorDescriptor {
-
- private static final int AGGREGATE_ACTIVITY_ID = 0;
-
- private static final int MERGE_ACTIVITY_ID = 1;
-
- private static final long serialVersionUID = 1L;
- private final int[] keyFields;
- private final IBinaryComparatorFactory[] comparatorFactories;
- private final INormalizedKeyComputerFactory firstNormalizerFactory;
-
- private final IAggregatorDescriptorFactory aggregatorFactory;
- private final IAggregatorDescriptorFactory mergerFactory;
-
- private final int framesLimit;
- private final ISpillableTableFactory spillableTableFactory;
- private final boolean isOutputSorted;
-
- public ExternalGroupOperatorDescriptor(JobSpecification spec, int[] keyFields, int framesLimit,
- IBinaryComparatorFactory[] comparatorFactories, INormalizedKeyComputerFactory firstNormalizerFactory,
- IAggregatorDescriptorFactory aggregatorFactory, IAggregatorDescriptorFactory mergerFactory,
- RecordDescriptor recordDescriptor, ISpillableTableFactory spillableTableFactory, boolean isOutputSorted) {
- super(spec, 1, 1);
- this.framesLimit = framesLimit;
- if (framesLimit <= 1) {
- /**
- * Minimum of 2 frames: 1 for input records, and 1 for output
- * aggregation results.
- */
- throw new IllegalStateException("frame limit should at least be 2, but it is " + framesLimit + "!");
- }
- this.aggregatorFactory = aggregatorFactory;
- this.mergerFactory = mergerFactory;
- this.keyFields = keyFields;
- this.comparatorFactories = comparatorFactories;
- this.firstNormalizerFactory = firstNormalizerFactory;
- this.spillableTableFactory = spillableTableFactory;
- this.isOutputSorted = isOutputSorted;
-
- /**
- * Set the record descriptor. Note that since this operator is a unary
- * operator, only the first record descriptor is used here.
- */
- recordDescriptors[0] = recordDescriptor;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor#contributeActivities
- * (edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder)
- */
- @Override
- public void contributeActivities(IActivityGraphBuilder builder) {
- AggregateActivity aggregateAct = new AggregateActivity(new ActivityId(getOperatorId(), AGGREGATE_ACTIVITY_ID));
- MergeActivity mergeAct = new MergeActivity(new ActivityId(odId, MERGE_ACTIVITY_ID));
-
- builder.addActivity(aggregateAct);
- builder.addSourceEdge(0, aggregateAct, 0);
-
- builder.addActivity(mergeAct);
- builder.addTargetEdge(0, mergeAct, 0);
-
- builder.addBlockingEdge(aggregateAct, mergeAct);
- }
-
- public static class AggregateActivityState extends AbstractTaskState {
- private LinkedList<RunFileReader> runs;
-
- private ISpillableTable gTable;
-
- public AggregateActivityState() {
- }
-
- private AggregateActivityState(JobId jobId, TaskId tId) {
- super(jobId, tId);
- }
-
- @Override
- public void toBytes(DataOutput out) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void fromBytes(DataInput in) throws IOException {
- throw new UnsupportedOperationException();
- }
- }
-
- private class AggregateActivity extends AbstractActivityNode {
- private static final long serialVersionUID = 1L;
-
- public AggregateActivity(ActivityId id) {
- super(id);
- }
-
- @Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
- final IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions)
- throws HyracksDataException {
- final FrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(),
- recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0));
- IOperatorNodePushable op = new AbstractUnaryInputSinkOperatorNodePushable() {
- private AggregateActivityState state;
-
- @Override
- public void open() throws HyracksDataException {
- state = new AggregateActivityState(ctx.getJobletContext().getJobId(), new TaskId(getActivityId(),
- partition));
- state.runs = new LinkedList<RunFileReader>();
- state.gTable = spillableTableFactory.buildSpillableTable(ctx, keyFields, comparatorFactories,
- firstNormalizerFactory, aggregatorFactory,
- recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0), recordDescriptors[0],
- ExternalGroupOperatorDescriptor.this.framesLimit);
- state.gTable.reset();
- }
-
- @Override
- public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
- accessor.reset(buffer);
- int tupleCount = accessor.getTupleCount();
- for (int i = 0; i < tupleCount; i++) {
- /**
- * If the group table is too large, flush the table into
- * a run file.
- */
- if (!state.gTable.insert(accessor, i)) {
- flushFramesToRun();
- if (!state.gTable.insert(accessor, i))
- throw new HyracksDataException(
- "Failed to insert a new buffer into the aggregate operator!");
- }
- }
- }
-
- @Override
- public void fail() throws HyracksDataException {
- throw new HyracksDataException("failed");
- }
-
- @Override
- public void close() throws HyracksDataException {
- if (state.gTable.getFrameCount() >= 0) {
- if (state.runs.size() > 0) {
- /**
- * flush the memory into the run file.
- */
- flushFramesToRun();
- state.gTable.close();
- state.gTable = null;
- }
- }
- ctx.setTaskState(state);
- }
-
- private void flushFramesToRun() throws HyracksDataException {
- FileReference runFile;
- try {
- runFile = ctx.getJobletContext().createManagedWorkspaceFile(
- ExternalGroupOperatorDescriptor.class.getSimpleName());
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- RunFileWriter writer = new RunFileWriter(runFile, ctx.getIOManager());
- writer.open();
- try {
- state.gTable.sortFrames();
- state.gTable.flushFrames(writer, true);
- } catch (Exception ex) {
- throw new HyracksDataException(ex);
- } finally {
- writer.close();
- }
- state.gTable.reset();
- state.runs.add(((RunFileWriter) writer).createReader());
- }
- };
- return op;
- }
- }
-
- private class MergeActivity extends AbstractActivityNode {
- private static final long serialVersionUID = 1L;
-
- public MergeActivity(ActivityId id) {
- super(id);
- }
-
- @Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions)
- throws HyracksDataException {
- final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
- for (int i = 0; i < comparatorFactories.length; ++i) {
- comparators[i] = comparatorFactories[i].createBinaryComparator();
- }
-
- int[] keyFieldsInPartialResults = new int[keyFields.length];
- for (int i = 0; i < keyFieldsInPartialResults.length; i++) {
- keyFieldsInPartialResults[i] = i;
- }
-
- final IAggregatorDescriptor aggregator = mergerFactory.createAggregator(ctx, recordDescriptors[0],
- recordDescriptors[0], keyFields, keyFieldsInPartialResults);
- final AggregateState aggregateState = aggregator.createAggregateStates();
-
- final int[] storedKeys = new int[keyFields.length];
- /**
- * Get the list of the fields in the stored records.
- */
- for (int i = 0; i < keyFields.length; ++i) {
- storedKeys[i] = i;
- }
-
- final ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(recordDescriptors[0].getFields().length);
-
- IOperatorNodePushable op = new AbstractUnaryOutputSourceOperatorNodePushable() {
- /**
- * Input frames, one for each run file.
- */
- private List<ByteBuffer> inFrames;
-
- /**
- * Output frame.
- */
- private ByteBuffer outFrame, writerFrame;
- private final FrameTupleAppender outAppender = new FrameTupleAppender(ctx.getFrameSize());
- private FrameTupleAppender writerAppender;
-
- private LinkedList<RunFileReader> runs;
-
- private AggregateActivityState aggState;
-
- private ArrayTupleBuilder finalTupleBuilder;
-
- /**
- * how many frames to be read ahead once
- */
- private int runFrameLimit = 1;
-
- private int[] currentFrameIndexInRun;
- private int[] currentRunFrames;
-
- private final FrameTupleAccessor outFrameAccessor = new FrameTupleAccessor(ctx.getFrameSize(),
- recordDescriptors[0]);
-
- public void initialize() throws HyracksDataException {
- aggState = (AggregateActivityState) ctx.getTaskState(new TaskId(new ActivityId(getOperatorId(),
- AGGREGATE_ACTIVITY_ID), partition));
- runs = aggState.runs;
- writer.open();
- try {
- if (runs.size() <= 0) {
- ISpillableTable gTable = aggState.gTable;
- if (gTable != null) {
- if (isOutputSorted)
- gTable.sortFrames();
- gTable.flushFrames(writer, false);
- }
- gTable = null;
- aggState = null;
- System.gc();
- } else {
- aggState = null;
- System.gc();
- runs = new LinkedList<RunFileReader>(runs);
- inFrames = new ArrayList<ByteBuffer>();
- outFrame = ctx.allocateFrame();
- outAppender.reset(outFrame, true);
- outFrameAccessor.reset(outFrame);
- while (runs.size() > 0) {
- try {
- doPass(runs);
- } catch (Exception e) {
- throw new HyracksDataException(e);
- }
- }
- inFrames.clear();
- }
- } catch (Exception e) {
- writer.fail();
- throw new HyracksDataException(e);
- } finally {
- aggregateState.close();
- writer.close();
- }
- }
-
- private void doPass(LinkedList<RunFileReader> runs) throws HyracksDataException {
- FileReference newRun = null;
- IFrameWriter writer = this.writer;
- boolean finalPass = false;
-
- while (inFrames.size() + 2 < framesLimit) {
- inFrames.add(ctx.allocateFrame());
- }
- int runNumber;
- if (runs.size() + 2 <= framesLimit) {
- finalPass = true;
- runFrameLimit = (framesLimit - 2) / runs.size();
- runNumber = runs.size();
- } else {
- runNumber = framesLimit - 2;
- newRun = ctx.getJobletContext().createManagedWorkspaceFile(
- ExternalGroupOperatorDescriptor.class.getSimpleName());
- writer = new RunFileWriter(newRun, ctx.getIOManager());
- writer.open();
- }
- try {
- currentFrameIndexInRun = new int[runNumber];
- currentRunFrames = new int[runNumber];
- /**
- * Create file readers for each input run file, only for
- * the ones fit into the inFrames
- */
- RunFileReader[] runFileReaders = new RunFileReader[runNumber];
- FrameTupleAccessor[] tupleAccessors = new FrameTupleAccessor[inFrames.size()];
- Comparator<ReferenceEntry> comparator = createEntryComparator(comparators);
- ReferencedPriorityQueue topTuples = new ReferencedPriorityQueue(ctx.getFrameSize(),
- recordDescriptors[0], runNumber, comparator);
- /**
- * current tuple index in each run
- */
- int[] tupleIndices = new int[runNumber];
-
- for (int i = 0; i < runNumber; i++) {
- int runIndex = topTuples.peek().getRunid();
- tupleIndices[runIndex] = 0;
- // Load the run file
- runFileReaders[runIndex] = runs.get(runIndex);
- runFileReaders[runIndex].open();
-
- currentRunFrames[runIndex] = 0;
- currentFrameIndexInRun[runIndex] = runIndex * runFrameLimit;
- for (int j = 0; j < runFrameLimit; j++) {
- int frameIndex = currentFrameIndexInRun[runIndex] + j;
- if (runFileReaders[runIndex].nextFrame(inFrames.get(frameIndex))) {
- tupleAccessors[frameIndex] = new FrameTupleAccessor(ctx.getFrameSize(),
- recordDescriptors[0]);
- tupleAccessors[frameIndex].reset(inFrames.get(frameIndex));
- currentRunFrames[runIndex]++;
- if (j == 0)
- setNextTopTuple(runIndex, tupleIndices, runFileReaders, tupleAccessors,
- topTuples);
- } else {
- break;
- }
- }
- }
-
- /**
- * Start merging
- */
- while (!topTuples.areRunsExhausted()) {
- /**
- * Get the top record
- */
- ReferenceEntry top = topTuples.peek();
- int tupleIndex = top.getTupleIndex();
- int runIndex = topTuples.peek().getRunid();
- FrameTupleAccessor fta = top.getAccessor();
-
- int currentTupleInOutFrame = outFrameAccessor.getTupleCount() - 1;
- if (currentTupleInOutFrame < 0
- || compareFrameTuples(fta, tupleIndex, outFrameAccessor, currentTupleInOutFrame) != 0) {
- /**
- * Initialize the first output record Reset the
- * tuple builder
- */
-
- tupleBuilder.reset();
-
- for(int k = 0; k < storedKeys.length; k++){
- tupleBuilder.addField(fta, tupleIndex, storedKeys[k]);
- }
-
- aggregator.init(tupleBuilder, fta, tupleIndex, aggregateState);
-
- if (!outAppender.appendSkipEmptyField(tupleBuilder.getFieldEndOffsets(),
- tupleBuilder.getByteArray(), 0, tupleBuilder.getSize())) {
- flushOutFrame(writer, finalPass);
- if (!outAppender.appendSkipEmptyField(tupleBuilder.getFieldEndOffsets(),
- tupleBuilder.getByteArray(), 0, tupleBuilder.getSize())) {
- throw new HyracksDataException(
- "The partial result is too large to be initialized in a frame.");
- }
- }
-
- } else {
- /**
- * if new tuple is in the same group of the
- * current aggregator do merge and output to the
- * outFrame
- */
-
- aggregator.aggregate(fta, tupleIndex, outFrameAccessor, currentTupleInOutFrame,
- aggregateState);
-
- }
- tupleIndices[runIndex]++;
- setNextTopTuple(runIndex, tupleIndices, runFileReaders, tupleAccessors, topTuples);
- }
-
- if (outAppender.getTupleCount() > 0) {
- flushOutFrame(writer, finalPass);
- outAppender.reset(outFrame, true);
- }
-
- aggregator.close();
-
- runs.subList(0, runNumber).clear();
- /**
- * insert the new run file into the beginning of the run
- * file list
- */
- if (!finalPass) {
- runs.add(0, ((RunFileWriter) writer).createReader());
- }
- } finally {
- if (!finalPass) {
- writer.close();
- }
- }
- }
-
- private void flushOutFrame(IFrameWriter writer, boolean isFinal) throws HyracksDataException {
-
- if (finalTupleBuilder == null) {
- finalTupleBuilder = new ArrayTupleBuilder(recordDescriptors[0].getFields().length);
- }
-
- if (writerFrame == null) {
- writerFrame = ctx.allocateFrame();
- }
-
- if (writerAppender == null) {
- writerAppender = new FrameTupleAppender(ctx.getFrameSize());
- writerAppender.reset(writerFrame, true);
- }
-
- outFrameAccessor.reset(outFrame);
-
- for (int i = 0; i < outFrameAccessor.getTupleCount(); i++) {
-
- finalTupleBuilder.reset();
-
- for (int k = 0; k < storedKeys.length; k++) {
- finalTupleBuilder.addField(outFrameAccessor, i, storedKeys[k]);
- }
-
- if (isFinal) {
-
- aggregator.outputFinalResult(finalTupleBuilder, outFrameAccessor, i, aggregateState);
-
- } else {
-
- aggregator.outputPartialResult(finalTupleBuilder, outFrameAccessor, i, aggregateState);
- }
-
- if (!writerAppender.appendSkipEmptyField(finalTupleBuilder.getFieldEndOffsets(),
- finalTupleBuilder.getByteArray(), 0, finalTupleBuilder.getSize())) {
- FrameUtils.flushFrame(writerFrame, writer);
- writerAppender.reset(writerFrame, true);
- if (!writerAppender.appendSkipEmptyField(finalTupleBuilder.getFieldEndOffsets(),
- finalTupleBuilder.getByteArray(), 0, finalTupleBuilder.getSize())) {
- throw new HyracksDataException(
- "Aggregation output is too large to be fit into a frame.");
- }
- }
- }
- if (writerAppender.getTupleCount() > 0) {
- FrameUtils.flushFrame(writerFrame, writer);
- writerAppender.reset(writerFrame, true);
- }
-
- outAppender.reset(outFrame, true);
- }
-
- private void setNextTopTuple(int runIndex, int[] tupleIndices, RunFileReader[] runCursors,
- FrameTupleAccessor[] tupleAccessors, ReferencedPriorityQueue topTuples)
- throws HyracksDataException {
- int runStart = runIndex * runFrameLimit;
- boolean existNext = false;
- if (tupleAccessors[currentFrameIndexInRun[runIndex]] == null || runCursors[runIndex] == null) {
- /**
- * run already closed
- */
- existNext = false;
- } else if (currentFrameIndexInRun[runIndex] - runStart < currentRunFrames[runIndex] - 1) {
- /**
- * not the last frame for this run
- */
- existNext = true;
- if (tupleIndices[runIndex] >= tupleAccessors[currentFrameIndexInRun[runIndex]].getTupleCount()) {
- tupleIndices[runIndex] = 0;
- currentFrameIndexInRun[runIndex]++;
- }
- } else if (tupleIndices[runIndex] < tupleAccessors[currentFrameIndexInRun[runIndex]]
- .getTupleCount()) {
- /**
- * the last frame has expired
- */
- existNext = true;
- } else {
- /**
- * If all tuples in the targeting frame have been
- * checked.
- */
- tupleIndices[runIndex] = 0;
- currentFrameIndexInRun[runIndex] = runStart;
- /**
- * read in batch
- */
- currentRunFrames[runIndex] = 0;
- for (int j = 0; j < runFrameLimit; j++) {
- int frameIndex = currentFrameIndexInRun[runIndex]
- + j;
- if (runCursors[runIndex].nextFrame(inFrames
- .get(frameIndex))) {
- tupleAccessors[frameIndex].reset(inFrames
- .get(frameIndex));
- existNext = true;
- currentRunFrames[runIndex]++;
- } else {
- break;
- }
- }
- }
-
- if (existNext) {
- topTuples.popAndReplace(tupleAccessors[currentFrameIndexInRun[runIndex]],
- tupleIndices[runIndex]);
- } else {
- topTuples.pop();
- closeRun(runIndex, runCursors, tupleAccessors);
- }
- }
-
- /**
- * Close the run file, and also the corresponding readers and
- * input frame.
- *
- * @param index
- * @param runCursors
- * @param tupleAccessor
- * @throws HyracksDataException
- */
- private void closeRun(int index, RunFileReader[] runCursors, IFrameTupleAccessor[] tupleAccessor)
- throws HyracksDataException {
- if (runCursors[index] != null) {
- runCursors[index].close();
- runCursors[index] = null;
- int frameOffset = index * runFrameLimit;
- for (int j = 0; j < runFrameLimit; j++) {
- tupleAccessor[frameOffset + j] = null;
- }
- }
- }
-
- private int compareFrameTuples(IFrameTupleAccessor fta1, int j1, IFrameTupleAccessor fta2, int j2) {
- byte[] b1 = fta1.getBuffer().array();
- byte[] b2 = fta2.getBuffer().array();
- for (int f = 0; f < keyFields.length; ++f) {
- int fIdx = f;
- int s1 = fta1.getTupleStartOffset(j1) + fta1.getFieldSlotsLength()
- + fta1.getFieldStartOffset(j1, fIdx);
- int l1 = fta1.getFieldLength(j1, fIdx);
- int s2 = fta2.getTupleStartOffset(j2) + fta2.getFieldSlotsLength()
- + fta2.getFieldStartOffset(j2, fIdx);
- int l2_start = fta2.getFieldStartOffset(j2, fIdx);
- int l2_end = fta2.getFieldEndOffset(j2, fIdx);
- int l2 = l2_end - l2_start;
- int c = comparators[f].compare(b1, s1, l1, b2, s2, l2);
- if (c != 0) {
- return c;
- }
- }
- return 0;
- }
- };
- return op;
- }
-
- private Comparator<ReferenceEntry> createEntryComparator(final IBinaryComparator[] comparators) {
- return new Comparator<ReferenceEntry>() {
-
- @Override
- public int compare(ReferenceEntry o1, ReferenceEntry o2) {
- FrameTupleAccessor fta1 = (FrameTupleAccessor) o1.getAccessor();
- FrameTupleAccessor fta2 = (FrameTupleAccessor) o2.getAccessor();
- int j1 = o1.getTupleIndex();
- int j2 = o2.getTupleIndex();
- byte[] b1 = fta1.getBuffer().array();
- byte[] b2 = fta2.getBuffer().array();
- for (int f = 0; f < keyFields.length; ++f) {
- int fIdx = f;
- int s1 = fta1.getTupleStartOffset(j1) + fta1.getFieldSlotsLength()
- + fta1.getFieldStartOffset(j1, fIdx);
- int l1 = fta1.getFieldEndOffset(j1, fIdx) - fta1.getFieldStartOffset(j1, fIdx);
- int s2 = fta2.getTupleStartOffset(j2) + fta2.getFieldSlotsLength()
- + fta2.getFieldStartOffset(j2, fIdx);
- int l2 = fta2.getFieldEndOffset(j2, fIdx) - fta2.getFieldStartOffset(j2, fIdx);
- int c = comparators[f].compare(b1, s1, l1, b2, s2, l2);
- if (c != 0) {
- return c;
- }
- }
- return 0;
- }
-
- };
- }
-
- }
-
-}
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashGroupOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashGroupOperatorDescriptor.java
deleted file mode 100644
index 49443d1..0000000
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashGroupOperatorDescriptor.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.group;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.ActivityId;
-import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
-import edu.uci.ics.hyracks.api.dataflow.TaskId;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractTaskState;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
-
-/**
- *
- */
-public class HashGroupOperatorDescriptor extends AbstractOperatorDescriptor {
-
- private static final int HASH_BUILD_ACTIVITY_ID = 0;
-
- private static final int OUTPUT_ACTIVITY_ID = 1;
-
- private static final long serialVersionUID = 1L;
-
- private final int[] keys;
- private final ITuplePartitionComputerFactory tpcf;
- private final IBinaryComparatorFactory[] comparatorFactories;
-
- private final IAggregatorDescriptorFactory aggregatorFactory;
-
- private final int tableSize;
-
- public HashGroupOperatorDescriptor(JobSpecification spec, int[] keys, ITuplePartitionComputerFactory tpcf,
- IBinaryComparatorFactory[] comparatorFactories, IAggregatorDescriptorFactory aggregatorFactory,
- RecordDescriptor outRecordDescriptor, int tableSize) {
- super(spec, 1, 1);
- this.keys = keys;
- this.tpcf = tpcf;
- this.comparatorFactories = comparatorFactories;
- this.aggregatorFactory = aggregatorFactory;
- recordDescriptors[0] = outRecordDescriptor;
- this.tableSize = tableSize;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor#contributeActivities
- * (edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder)
- */
- @Override
- public void contributeActivities(IActivityGraphBuilder builder) {
- HashBuildActivity ha = new HashBuildActivity(new ActivityId(odId, HASH_BUILD_ACTIVITY_ID));
- builder.addActivity(ha);
-
- OutputActivity oa = new OutputActivity(new ActivityId(odId, OUTPUT_ACTIVITY_ID));
- builder.addActivity(oa);
-
- builder.addSourceEdge(0, ha, 0);
- builder.addTargetEdge(0, oa, 0);
- builder.addBlockingEdge(ha, oa);
- }
-
- public static class HashBuildActivityState extends AbstractTaskState {
- private GroupingHashTable table;
-
- public HashBuildActivityState() {
- }
-
- private HashBuildActivityState(JobId jobId, TaskId tId) {
- super(jobId, tId);
- }
-
- @Override
- public void toBytes(DataOutput out) throws IOException {
-
- }
-
- @Override
- public void fromBytes(DataInput in) throws IOException {
-
- }
- }
-
- private class HashBuildActivity extends AbstractActivityNode {
- private static final long serialVersionUID = 1L;
-
- public HashBuildActivity(ActivityId id) {
- super(id);
- }
-
- @Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
- final IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
- final FrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(),
- recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0));
- return new AbstractUnaryInputSinkOperatorNodePushable() {
- private HashBuildActivityState state;
-
- @Override
- public void open() throws HyracksDataException {
- state = new HashBuildActivityState(ctx.getJobletContext().getJobId(), new TaskId(getActivityId(),
- partition));
- state.table = new GroupingHashTable(ctx, keys, comparatorFactories, tpcf, aggregatorFactory,
- recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0), recordDescriptors[0],
- tableSize);
- }
-
- @Override
- public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
- accessor.reset(buffer);
- int tupleCount = accessor.getTupleCount();
- for (int i = 0; i < tupleCount; ++i) {
- try {
- state.table.insert(accessor, i);
- } catch (Exception e) {
- System.out.println(e.toString());
- throw new HyracksDataException(e);
- }
- }
- }
-
- @Override
- public void close() throws HyracksDataException {
- ctx.setTaskState(state);
- }
-
- @Override
- public void fail() throws HyracksDataException {
- throw new HyracksDataException("HashGroupOperator is failed.");
- }
- };
- }
- }
-
- private class OutputActivity extends AbstractActivityNode {
- private static final long serialVersionUID = 1L;
-
- public OutputActivity(ActivityId id) {
- super(id);
- }
-
- @Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
- return new AbstractUnaryOutputSourceOperatorNodePushable() {
- @Override
- public void initialize() throws HyracksDataException {
- HashBuildActivityState buildState = (HashBuildActivityState) ctx.getTaskState(new TaskId(
- new ActivityId(getOperatorId(), HASH_BUILD_ACTIVITY_ID), partition));
- GroupingHashTable table = buildState.table;
- writer.open();
- try {
- table.write(writer);
- } catch (Exception e) {
- writer.fail();
- throw new HyracksDataException(e);
- } finally {
- writer.close();
- }
- }
- };
- }
- }
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashSpillableTableFactory.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashSpillableTableFactory.java
index b0e092d..f2b56fa 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashSpillableTableFactory.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashSpillableTableFactory.java
@@ -27,7 +27,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -80,25 +79,10 @@
}
RecordDescriptor internalRecordDescriptor = outRecordDescriptor;
- final FrameTupleAccessor storedKeysAccessor1;
- final FrameTupleAccessor storedKeysAccessor2;
- if (keyFields.length >= outRecordDescriptor.getFields().length) {
- // for the case of zero-aggregations
- ISerializerDeserializer<?>[] fields = outRecordDescriptor.getFields();
- ITypeTraits[] types = outRecordDescriptor.getTypeTraits();
- ISerializerDeserializer<?>[] newFields = new ISerializerDeserializer[fields.length + 1];
- for (int i = 0; i < fields.length; i++)
- newFields[i] = fields[i];
- ITypeTraits[] newTypes = null;
- if (types != null) {
- newTypes = new ITypeTraits[types.length + 1];
- for (int i = 0; i < types.length; i++)
- newTypes[i] = types[i];
- }
- internalRecordDescriptor = new RecordDescriptor(newFields, newTypes);
- }
- storedKeysAccessor1 = new FrameTupleAccessor(ctx.getFrameSize(), internalRecordDescriptor);
- storedKeysAccessor2 = new FrameTupleAccessor(ctx.getFrameSize(), internalRecordDescriptor);
+ final FrameTupleAccessor storedKeysAccessor1 = new FrameTupleAccessor(ctx.getFrameSize(),
+ internalRecordDescriptor);
+ final FrameTupleAccessor storedKeysAccessor2 = new FrameTupleAccessor(ctx.getFrameSize(),
+ internalRecordDescriptor);
final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
for (int i = 0; i < comparatorFactories.length; ++i) {
@@ -225,7 +209,7 @@
for (int k = 0; k < keyFields.length; k++) {
stateTupleBuilder.addField(accessor, tIndex, keyFields[k]);
}
-
+
aggregator.init(stateTupleBuilder, accessor, tIndex, aggregateState);
if (!stateAppender.appendSkipEmptyField(stateTupleBuilder.getFieldEndOffsets(),
stateTupleBuilder.getByteArray(), 0, stateTupleBuilder.getSize())) {
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/PreclusteredGroupOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/PreclusteredGroupOperatorDescriptor.java
deleted file mode 100644
index c718ee4..0000000
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/PreclusteredGroupOperatorDescriptor.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.group;
-
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
-
-public class PreclusteredGroupOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
- private final int[] groupFields;
- private final IBinaryComparatorFactory[] comparatorFactories;
- private final IAggregatorDescriptorFactory aggregatorFactory;
-
- private static final long serialVersionUID = 1L;
-
- public PreclusteredGroupOperatorDescriptor(JobSpecification spec, int[] groupFields,
- IBinaryComparatorFactory[] comparatorFactories, IAggregatorDescriptorFactory aggregatorFactory,
- RecordDescriptor recordDescriptor) {
- super(spec, 1, 1);
- this.groupFields = groupFields;
- this.comparatorFactories = comparatorFactories;
- this.aggregatorFactory = aggregatorFactory;
- recordDescriptors[0] = recordDescriptor;
- }
-
- @Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
- final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
- for (int i = 0; i < comparatorFactories.length; ++i) {
- comparators[i] = comparatorFactories[i].createBinaryComparator();
- }
- final RecordDescriptor inRecordDesc = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0);
- final IAggregatorDescriptor aggregator = aggregatorFactory.createAggregator(ctx, inRecordDesc,
- recordDescriptors[0], groupFields, groupFields);
- final ByteBuffer copyFrame = ctx.allocateFrame();
- final FrameTupleAccessor copyFrameAccessor = new FrameTupleAccessor(ctx.getFrameSize(), inRecordDesc);
- copyFrameAccessor.reset(copyFrame);
- ByteBuffer outFrame = ctx.allocateFrame();
- final FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- appender.reset(outFrame, true);
- return new AbstractUnaryInputUnaryOutputOperatorNodePushable() {
- private PreclusteredGroupWriter pgw;
-
- @Override
- public void open() throws HyracksDataException {
- pgw = new PreclusteredGroupWriter(ctx, groupFields, comparators, aggregator, inRecordDesc,
- recordDescriptors[0], writer);
- pgw.open();
- }
-
- @Override
- public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
- pgw.nextFrame(buffer);
- }
-
- @Override
- public void fail() throws HyracksDataException {
- pgw.fail();
- }
-
- @Override
- public void close() throws HyracksDataException {
- pgw.close();
- }
- };
- }
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/external/ExternalGroupBuildOperatorNodePushable.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/external/ExternalGroupBuildOperatorNodePushable.java
new file mode 100644
index 0000000..ede6728
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/external/ExternalGroupBuildOperatorNodePushable.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group.external;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.LinkedList;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.io.RunFileReader;
+import edu.uci.ics.hyracks.dataflow.common.io.RunFileWriter;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.ISpillableTable;
+import edu.uci.ics.hyracks.dataflow.std.group.ISpillableTableFactory;
+
+class ExternalGroupBuildOperatorNodePushable extends AbstractUnaryInputSinkOperatorNodePushable {
+ private final IHyracksTaskContext ctx;
+ private final Object stateId;
+ private final int[] keyFields;
+ private final IBinaryComparatorFactory[] comparatorFactories;
+ private final INormalizedKeyComputerFactory firstNormalizerFactory;
+ private final IAggregatorDescriptorFactory aggregatorFactory;
+ private final int framesLimit;
+ private final ISpillableTableFactory spillableTableFactory;
+ private final RecordDescriptor inRecordDescriptor;
+ private final RecordDescriptor outRecordDescriptor;
+ private final FrameTupleAccessor accessor;
+
+ private ExternalGroupState state;
+
+ ExternalGroupBuildOperatorNodePushable(IHyracksTaskContext ctx, Object stateId, int[] keyFields, int framesLimit,
+ IBinaryComparatorFactory[] comparatorFactories, INormalizedKeyComputerFactory firstNormalizerFactory,
+ IAggregatorDescriptorFactory aggregatorFactory, RecordDescriptor inRecordDescriptor,
+ RecordDescriptor outRecordDescriptor, ISpillableTableFactory spillableTableFactory) {
+ this.ctx = ctx;
+ this.stateId = stateId;
+ this.framesLimit = framesLimit;
+ this.aggregatorFactory = aggregatorFactory;
+ this.keyFields = keyFields;
+ this.comparatorFactories = comparatorFactories;
+ this.firstNormalizerFactory = firstNormalizerFactory;
+ this.spillableTableFactory = spillableTableFactory;
+ this.inRecordDescriptor = inRecordDescriptor;
+ this.outRecordDescriptor = outRecordDescriptor;
+ this.accessor = new FrameTupleAccessor(ctx.getFrameSize(), inRecordDescriptor);
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ state = new ExternalGroupState(ctx.getJobletContext().getJobId(), stateId);
+ state.setRuns(new LinkedList<RunFileReader>());
+ ISpillableTable table = spillableTableFactory.buildSpillableTable(ctx, keyFields, comparatorFactories,
+ firstNormalizerFactory, aggregatorFactory, inRecordDescriptor, outRecordDescriptor, framesLimit);
+ table.reset();
+ state.setSpillableTable(table);
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ accessor.reset(buffer);
+ int tupleCount = accessor.getTupleCount();
+ ISpillableTable gTable = state.getSpillableTable();
+ for (int i = 0; i < tupleCount; i++) {
+ /**
+ * If the group table is too large, flush the table into
+ * a run file.
+ */
+ if (!gTable.insert(accessor, i)) {
+ flushFramesToRun();
+ if (!gTable.insert(accessor, i))
+ throw new HyracksDataException("Failed to insert a new buffer into the aggregate operator!");
+ }
+ }
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ //do nothing for failures
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ ISpillableTable gTable = state.getSpillableTable();
+ if (gTable.getFrameCount() >= 0) {
+ if (state.getRuns().size() > 0) {
+ /**
+ * flush the memory into the run file.
+ */
+ flushFramesToRun();
+ gTable.close();
+ gTable = null;
+ }
+ }
+ ctx.setStateObject(state);
+ }
+
+ private void flushFramesToRun() throws HyracksDataException {
+ FileReference runFile;
+ try {
+ runFile = ctx.getJobletContext().createManagedWorkspaceFile(
+ ExternalGroupOperatorDescriptor.class.getSimpleName());
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ RunFileWriter writer = new RunFileWriter(runFile, ctx.getIOManager());
+ writer.open();
+ ISpillableTable gTable = state.getSpillableTable();
+ try {
+ gTable.sortFrames();
+ gTable.flushFrames(writer, true);
+ } catch (Exception ex) {
+ throw new HyracksDataException(ex);
+ } finally {
+ writer.close();
+ }
+ gTable.reset();
+ state.getRuns().add(((RunFileWriter) writer).createReader());
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/external/ExternalGroupMergeOperatorNodePushable.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/external/ExternalGroupMergeOperatorNodePushable.java
new file mode 100644
index 0000000..297b75c
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/external/ExternalGroupMergeOperatorNodePushable.java
@@ -0,0 +1,462 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group.external;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+import edu.uci.ics.hyracks.dataflow.common.io.RunFileReader;
+import edu.uci.ics.hyracks.dataflow.common.io.RunFileWriter;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
+import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.ISpillableTable;
+import edu.uci.ics.hyracks.dataflow.std.util.ReferenceEntry;
+import edu.uci.ics.hyracks.dataflow.std.util.ReferencedPriorityQueue;
+
+class ExternalGroupMergeOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable {
+ private final IHyracksTaskContext ctx;
+ private final Object stateId;
+ private final int[] keyFields;
+ private final IBinaryComparator[] comparators;
+ private final AggregateState aggregateState;
+ private final ArrayTupleBuilder tupleBuilder;
+ private final int[] storedKeys;
+ private final IAggregatorDescriptor aggregator;
+ private final boolean isOutputSorted;
+ private final int framesLimit;
+ private final RecordDescriptor outRecordDescriptor;
+ /**
+ * Input frames, one for each run file.
+ */
+ private List<ByteBuffer> inFrames;
+ /**
+ * Output frame.
+ */
+ private ByteBuffer outFrame, writerFrame;
+ private final FrameTupleAppender outAppender;
+ private FrameTupleAppender writerAppender;
+ private LinkedList<RunFileReader> runs;
+ private ExternalGroupState aggState;
+ private ArrayTupleBuilder finalTupleBuilder;
+ /**
+ * how many frames to be read ahead once
+ */
+ private int runFrameLimit = 1;
+ private int[] currentFrameIndexInRun;
+ private int[] currentRunFrames;
+ private final FrameTupleAccessor outFrameAccessor;
+
+ ExternalGroupMergeOperatorNodePushable(IHyracksTaskContext ctx, Object stateId,
+ IBinaryComparatorFactory[] comparatorFactories, int[] keyFields,
+ IAggregatorDescriptorFactory mergerFactory, boolean isOutputSorted, int framesLimit,
+ RecordDescriptor outRecordDescriptor) throws HyracksDataException {
+ this.stateId = stateId;
+ this.keyFields = keyFields;
+ comparators = new IBinaryComparator[comparatorFactories.length];
+ for (int i = 0; i < comparatorFactories.length; ++i) {
+ comparators[i] = comparatorFactories[i].createBinaryComparator();
+ }
+ int[] keyFieldsInPartialResults = new int[keyFields.length];
+ for (int i = 0; i < keyFieldsInPartialResults.length; i++) {
+ keyFieldsInPartialResults[i] = i;
+ }
+
+ aggregator = mergerFactory.createAggregator(ctx, outRecordDescriptor, outRecordDescriptor, keyFields,
+ keyFieldsInPartialResults);
+ aggregateState = aggregator.createAggregateStates();
+
+ storedKeys = new int[keyFields.length];
+ /**
+ * Get the list of the fields in the stored records.
+ */
+ for (int i = 0; i < keyFields.length; ++i) {
+ storedKeys[i] = i;
+ }
+
+ tupleBuilder = new ArrayTupleBuilder(outRecordDescriptor.getFields().length);
+ this.ctx = ctx;
+ outAppender = new FrameTupleAppender(ctx.getFrameSize());
+ outFrameAccessor = new FrameTupleAccessor(ctx.getFrameSize(), outRecordDescriptor);
+ this.isOutputSorted = isOutputSorted;
+ this.framesLimit = framesLimit;
+ this.outRecordDescriptor = outRecordDescriptor;
+ }
+
+ public void initialize() throws HyracksDataException {
+ aggState = (ExternalGroupState) ctx.getStateObject(stateId);
+ runs = aggState.getRuns();
+ writer.open();
+ try {
+ if (runs.size() <= 0) {
+ ISpillableTable gTable = aggState.getSpillableTable();
+ if (gTable != null) {
+ if (isOutputSorted)
+ gTable.sortFrames();
+ gTable.flushFrames(writer, false);
+ }
+ gTable = null;
+ aggState = null;
+ } else {
+ aggState = null;
+ runs = new LinkedList<RunFileReader>(runs);
+ inFrames = new ArrayList<ByteBuffer>();
+ outFrame = ctx.allocateFrame();
+ outAppender.reset(outFrame, true);
+ outFrameAccessor.reset(outFrame);
+ while (runs.size() > 0) {
+ try {
+ doPass(runs);
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ inFrames.clear();
+ }
+ } catch (Exception e) {
+ writer.fail();
+ throw new HyracksDataException(e);
+ } finally {
+ aggregateState.close();
+ writer.close();
+ }
+ }
+
+ private void doPass(LinkedList<RunFileReader> runs) throws HyracksDataException {
+ FileReference newRun = null;
+ IFrameWriter writer = this.writer;
+ boolean finalPass = false;
+
+ while (inFrames.size() + 2 < framesLimit) {
+ inFrames.add(ctx.allocateFrame());
+ }
+ int runNumber;
+ if (runs.size() + 2 <= framesLimit) {
+ finalPass = true;
+ runFrameLimit = (framesLimit - 2) / runs.size();
+ runNumber = runs.size();
+ } else {
+ runNumber = framesLimit - 2;
+ newRun = ctx.getJobletContext().createManagedWorkspaceFile(
+ ExternalGroupOperatorDescriptor.class.getSimpleName());
+ writer = new RunFileWriter(newRun, ctx.getIOManager());
+ writer.open();
+ }
+ try {
+ currentFrameIndexInRun = new int[runNumber];
+ currentRunFrames = new int[runNumber];
+ /**
+ * Create file readers for each input run file, only for
+ * the ones fit into the inFrames
+ */
+ RunFileReader[] runFileReaders = new RunFileReader[runNumber];
+ FrameTupleAccessor[] tupleAccessors = new FrameTupleAccessor[inFrames.size()];
+ Comparator<ReferenceEntry> comparator = createEntryComparator(comparators);
+ ReferencedPriorityQueue topTuples = new ReferencedPriorityQueue(ctx.getFrameSize(), outRecordDescriptor,
+ runNumber, comparator);
+ /**
+ * current tuple index in each run
+ */
+ int[] tupleIndices = new int[runNumber];
+
+ for (int i = 0; i < runNumber; i++) {
+ int runIndex = topTuples.peek().getRunid();
+ tupleIndices[runIndex] = 0;
+ // Load the run file
+ runFileReaders[runIndex] = runs.get(runIndex);
+ runFileReaders[runIndex].open();
+
+ currentRunFrames[runIndex] = 0;
+ currentFrameIndexInRun[runIndex] = runIndex * runFrameLimit;
+ for (int j = 0; j < runFrameLimit; j++) {
+ int frameIndex = currentFrameIndexInRun[runIndex] + j;
+ if (runFileReaders[runIndex].nextFrame(inFrames.get(frameIndex))) {
+ tupleAccessors[frameIndex] = new FrameTupleAccessor(ctx.getFrameSize(), outRecordDescriptor);
+ tupleAccessors[frameIndex].reset(inFrames.get(frameIndex));
+ currentRunFrames[runIndex]++;
+ if (j == 0)
+ setNextTopTuple(runIndex, tupleIndices, runFileReaders, tupleAccessors, topTuples);
+ } else {
+ break;
+ }
+ }
+ }
+
+ /**
+ * Start merging
+ */
+ while (!topTuples.areRunsExhausted()) {
+ /**
+ * Get the top record
+ */
+ ReferenceEntry top = topTuples.peek();
+ int tupleIndex = top.getTupleIndex();
+ int runIndex = topTuples.peek().getRunid();
+ FrameTupleAccessor fta = top.getAccessor();
+
+ int currentTupleInOutFrame = outFrameAccessor.getTupleCount() - 1;
+ if (currentTupleInOutFrame < 0
+ || compareFrameTuples(fta, tupleIndex, outFrameAccessor, currentTupleInOutFrame) != 0) {
+ /**
+ * Initialize the first output record Reset the
+ * tuple builder
+ */
+
+ tupleBuilder.reset();
+
+ for (int k = 0; k < storedKeys.length; k++) {
+ tupleBuilder.addField(fta, tupleIndex, storedKeys[k]);
+ }
+
+ aggregator.init(tupleBuilder, fta, tupleIndex, aggregateState);
+
+ if (!outAppender.appendSkipEmptyField(tupleBuilder.getFieldEndOffsets(),
+ tupleBuilder.getByteArray(), 0, tupleBuilder.getSize())) {
+ flushOutFrame(writer, finalPass);
+ if (!outAppender.appendSkipEmptyField(tupleBuilder.getFieldEndOffsets(),
+ tupleBuilder.getByteArray(), 0, tupleBuilder.getSize())) {
+ throw new HyracksDataException(
+ "The partial result is too large to be initialized in a frame.");
+ }
+ }
+
+ } else {
+ /**
+ * if new tuple is in the same group of the
+ * current aggregator do merge and output to the
+ * outFrame
+ */
+
+ aggregator.aggregate(fta, tupleIndex, outFrameAccessor, currentTupleInOutFrame, aggregateState);
+
+ }
+ tupleIndices[runIndex]++;
+ setNextTopTuple(runIndex, tupleIndices, runFileReaders, tupleAccessors, topTuples);
+ }
+
+ if (outAppender.getTupleCount() > 0) {
+ flushOutFrame(writer, finalPass);
+ outAppender.reset(outFrame, true);
+ }
+
+ aggregator.close();
+
+ runs.subList(0, runNumber).clear();
+ /**
+ * insert the new run file into the beginning of the run
+ * file list
+ */
+ if (!finalPass) {
+ runs.add(0, ((RunFileWriter) writer).createReader());
+ }
+ } finally {
+ if (!finalPass) {
+ writer.close();
+ }
+ }
+ }
+
+ private void flushOutFrame(IFrameWriter writer, boolean isFinal) throws HyracksDataException {
+
+ if (finalTupleBuilder == null) {
+ finalTupleBuilder = new ArrayTupleBuilder(outRecordDescriptor.getFields().length);
+ }
+
+ if (writerFrame == null) {
+ writerFrame = ctx.allocateFrame();
+ }
+
+ if (writerAppender == null) {
+ writerAppender = new FrameTupleAppender(ctx.getFrameSize());
+ writerAppender.reset(writerFrame, true);
+ }
+
+ outFrameAccessor.reset(outFrame);
+
+ for (int i = 0; i < outFrameAccessor.getTupleCount(); i++) {
+
+ finalTupleBuilder.reset();
+
+ for (int k = 0; k < storedKeys.length; k++) {
+ finalTupleBuilder.addField(outFrameAccessor, i, storedKeys[k]);
+ }
+
+ if (isFinal) {
+
+ aggregator.outputFinalResult(finalTupleBuilder, outFrameAccessor, i, aggregateState);
+
+ } else {
+
+ aggregator.outputPartialResult(finalTupleBuilder, outFrameAccessor, i, aggregateState);
+ }
+
+ if (!writerAppender.appendSkipEmptyField(finalTupleBuilder.getFieldEndOffsets(),
+ finalTupleBuilder.getByteArray(), 0, finalTupleBuilder.getSize())) {
+ FrameUtils.flushFrame(writerFrame, writer);
+ writerAppender.reset(writerFrame, true);
+ if (!writerAppender.appendSkipEmptyField(finalTupleBuilder.getFieldEndOffsets(),
+ finalTupleBuilder.getByteArray(), 0, finalTupleBuilder.getSize())) {
+ throw new HyracksDataException("Aggregation output is too large to be fit into a frame.");
+ }
+ }
+ }
+ if (writerAppender.getTupleCount() > 0) {
+ FrameUtils.flushFrame(writerFrame, writer);
+ writerAppender.reset(writerFrame, true);
+ }
+
+ outAppender.reset(outFrame, true);
+ }
+
+ private void setNextTopTuple(int runIndex, int[] tupleIndices, RunFileReader[] runCursors,
+ FrameTupleAccessor[] tupleAccessors, ReferencedPriorityQueue topTuples) throws HyracksDataException {
+ int runStart = runIndex * runFrameLimit;
+ boolean existNext = false;
+ if (tupleAccessors[currentFrameIndexInRun[runIndex]] == null || runCursors[runIndex] == null) {
+ /**
+ * run already closed
+ */
+ existNext = false;
+ } else if (currentFrameIndexInRun[runIndex] - runStart < currentRunFrames[runIndex] - 1) {
+ /**
+ * not the last frame for this run
+ */
+ existNext = true;
+ if (tupleIndices[runIndex] >= tupleAccessors[currentFrameIndexInRun[runIndex]].getTupleCount()) {
+ tupleIndices[runIndex] = 0;
+ currentFrameIndexInRun[runIndex]++;
+ }
+ } else if (tupleIndices[runIndex] < tupleAccessors[currentFrameIndexInRun[runIndex]].getTupleCount()) {
+ /**
+ * the last frame has expired
+ */
+ existNext = true;
+ } else {
+ /**
+ * If all tuples in the targeting frame have been
+ * checked.
+ */
+ tupleIndices[runIndex] = 0;
+ currentFrameIndexInRun[runIndex] = runStart;
+ /**
+ * read in batch
+ */
+ currentRunFrames[runIndex] = 0;
+ for (int j = 0; j < runFrameLimit; j++) {
+ int frameIndex = currentFrameIndexInRun[runIndex] + j;
+ if (runCursors[runIndex].nextFrame(inFrames.get(frameIndex))) {
+ tupleAccessors[frameIndex].reset(inFrames.get(frameIndex));
+ existNext = true;
+ currentRunFrames[runIndex]++;
+ } else {
+ break;
+ }
+ }
+ }
+
+ if (existNext) {
+ topTuples.popAndReplace(tupleAccessors[currentFrameIndexInRun[runIndex]], tupleIndices[runIndex]);
+ } else {
+ topTuples.pop();
+ closeRun(runIndex, runCursors, tupleAccessors);
+ }
+ }
+
+ /**
+ * Close the run file, and also the corresponding readers and
+ * input frame.
+ *
+ * @param index
+ * @param runCursors
+ * @param tupleAccessor
+ * @throws HyracksDataException
+ */
+ private void closeRun(int index, RunFileReader[] runCursors, IFrameTupleAccessor[] tupleAccessor)
+ throws HyracksDataException {
+ if (runCursors[index] != null) {
+ runCursors[index].close();
+ runCursors[index] = null;
+ int frameOffset = index * runFrameLimit;
+ for (int j = 0; j < runFrameLimit; j++) {
+ tupleAccessor[frameOffset + j] = null;
+ }
+ }
+ }
+
+ private int compareFrameTuples(IFrameTupleAccessor fta1, int j1, IFrameTupleAccessor fta2, int j2) {
+ byte[] b1 = fta1.getBuffer().array();
+ byte[] b2 = fta2.getBuffer().array();
+ for (int f = 0; f < keyFields.length; ++f) {
+ int fIdx = f;
+ int s1 = fta1.getTupleStartOffset(j1) + fta1.getFieldSlotsLength() + fta1.getFieldStartOffset(j1, fIdx);
+ int l1 = fta1.getFieldLength(j1, fIdx);
+ int s2 = fta2.getTupleStartOffset(j2) + fta2.getFieldSlotsLength() + fta2.getFieldStartOffset(j2, fIdx);
+ int l2_start = fta2.getFieldStartOffset(j2, fIdx);
+ int l2_end = fta2.getFieldEndOffset(j2, fIdx);
+ int l2 = l2_end - l2_start;
+ int c = comparators[f].compare(b1, s1, l1, b2, s2, l2);
+ if (c != 0) {
+ return c;
+ }
+ }
+ return 0;
+ }
+
+ private Comparator<ReferenceEntry> createEntryComparator(final IBinaryComparator[] comparators) {
+ return new Comparator<ReferenceEntry>() {
+
+ @Override
+ public int compare(ReferenceEntry o1, ReferenceEntry o2) {
+ FrameTupleAccessor fta1 = (FrameTupleAccessor) o1.getAccessor();
+ FrameTupleAccessor fta2 = (FrameTupleAccessor) o2.getAccessor();
+ int j1 = o1.getTupleIndex();
+ int j2 = o2.getTupleIndex();
+ byte[] b1 = fta1.getBuffer().array();
+ byte[] b2 = fta2.getBuffer().array();
+ for (int f = 0; f < keyFields.length; ++f) {
+ int fIdx = f;
+ int s1 = fta1.getTupleStartOffset(j1) + fta1.getFieldSlotsLength()
+ + fta1.getFieldStartOffset(j1, fIdx);
+ int l1 = fta1.getFieldEndOffset(j1, fIdx) - fta1.getFieldStartOffset(j1, fIdx);
+ int s2 = fta2.getTupleStartOffset(j2) + fta2.getFieldSlotsLength()
+ + fta2.getFieldStartOffset(j2, fIdx);
+ int l2 = fta2.getFieldEndOffset(j2, fIdx) - fta2.getFieldStartOffset(j2, fIdx);
+ int c = comparators[f].compare(b1, s1, l1, b2, s2, l2);
+ if (c != 0) {
+ return c;
+ }
+ }
+ return 0;
+ }
+
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/external/ExternalGroupOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/external/ExternalGroupOperatorDescriptor.java
new file mode 100644
index 0000000..9fd45e9
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/external/ExternalGroupOperatorDescriptor.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group.external;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.ActivityId;
+import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.TaskId;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.ISpillableTableFactory;
+
+/**
+ *
+ */
+public class ExternalGroupOperatorDescriptor extends AbstractOperatorDescriptor {
+
+ private static final int AGGREGATE_ACTIVITY_ID = 0;
+
+ private static final int MERGE_ACTIVITY_ID = 1;
+
+ private static final long serialVersionUID = 1L;
+ private final int[] keyFields;
+ private final IBinaryComparatorFactory[] comparatorFactories;
+ private final INormalizedKeyComputerFactory firstNormalizerFactory;
+
+ private final IAggregatorDescriptorFactory aggregatorFactory;
+ private final IAggregatorDescriptorFactory mergerFactory;
+
+ private final int framesLimit;
+ private final ISpillableTableFactory spillableTableFactory;
+ private final boolean isOutputSorted;
+
+ public ExternalGroupOperatorDescriptor(IOperatorDescriptorRegistry spec, int[] keyFields, int framesLimit,
+ IBinaryComparatorFactory[] comparatorFactories, INormalizedKeyComputerFactory firstNormalizerFactory,
+ IAggregatorDescriptorFactory aggregatorFactory, IAggregatorDescriptorFactory mergerFactory,
+ RecordDescriptor recordDescriptor, ISpillableTableFactory spillableTableFactory, boolean isOutputSorted) {
+ super(spec, 1, 1);
+ this.framesLimit = framesLimit;
+ if (framesLimit <= 1) {
+ /**
+ * Minimum of 2 frames: 1 for input records, and 1 for output
+ * aggregation results.
+ */
+ throw new IllegalStateException("frame limit should at least be 2, but it is " + framesLimit + "!");
+ }
+ this.aggregatorFactory = aggregatorFactory;
+ this.mergerFactory = mergerFactory;
+ this.keyFields = keyFields;
+ this.comparatorFactories = comparatorFactories;
+ this.firstNormalizerFactory = firstNormalizerFactory;
+ this.spillableTableFactory = spillableTableFactory;
+ this.isOutputSorted = isOutputSorted;
+
+ /**
+ * Set the record descriptor. Note that since this operator is a unary
+ * operator, only the first record descriptor is used here.
+ */
+ recordDescriptors[0] = recordDescriptor;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor#contributeActivities
+ * (edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder)
+ */
+ @Override
+ public void contributeActivities(IActivityGraphBuilder builder) {
+ AggregateActivity aggregateAct = new AggregateActivity(new ActivityId(getOperatorId(), AGGREGATE_ACTIVITY_ID));
+ MergeActivity mergeAct = new MergeActivity(new ActivityId(odId, MERGE_ACTIVITY_ID));
+
+ builder.addActivity(this, aggregateAct);
+ builder.addSourceEdge(0, aggregateAct, 0);
+
+ builder.addActivity(this, mergeAct);
+ builder.addTargetEdge(0, mergeAct, 0);
+
+ builder.addBlockingEdge(aggregateAct, mergeAct);
+ }
+
+ private class AggregateActivity extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ public AggregateActivity(ActivityId id) {
+ super(id);
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ final IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions)
+ throws HyracksDataException {
+ return new ExternalGroupBuildOperatorNodePushable(ctx, new TaskId(getActivityId(), partition), keyFields,
+ framesLimit, comparatorFactories, firstNormalizerFactory, aggregatorFactory,
+ recordDescProvider.getInputRecordDescriptor(getActivityId(), 0), recordDescriptors[0],
+ spillableTableFactory);
+ }
+ }
+
+ private class MergeActivity extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ public MergeActivity(ActivityId id) {
+ super(id);
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions)
+ throws HyracksDataException {
+ return new ExternalGroupMergeOperatorNodePushable(ctx, new TaskId(new ActivityId(getOperatorId(),
+ AGGREGATE_ACTIVITY_ID), partition), comparatorFactories, keyFields, mergerFactory, isOutputSorted,
+ framesLimit, recordDescriptors[0]);
+ }
+
+ }
+
+}
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/external/ExternalGroupState.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/external/ExternalGroupState.java
new file mode 100644
index 0000000..a3cf0e3
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/external/ExternalGroupState.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group.external;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.LinkedList;
+
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.dataflow.common.io.RunFileReader;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractStateObject;
+import edu.uci.ics.hyracks.dataflow.std.group.ISpillableTable;
+
+public class ExternalGroupState extends AbstractStateObject {
+ private LinkedList<RunFileReader> runs;
+
+ private ISpillableTable gTable;
+
+ public ExternalGroupState() {
+ }
+
+ ExternalGroupState(JobId jobId, Object id) {
+ super(jobId, id);
+ }
+
+ public LinkedList<RunFileReader> getRuns() {
+ return runs;
+ }
+
+ public void setRuns(LinkedList<RunFileReader> runs) {
+ this.runs = runs;
+ }
+
+ public ISpillableTable getSpillableTable() {
+ return gTable;
+ }
+
+ public void setSpillableTable(ISpillableTable gTable) {
+ this.gTable = gTable;
+ }
+
+ @Override
+ public void toBytes(DataOutput out) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void fromBytes(DataInput in) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/GroupingHashTable.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/hash/GroupingHashTable.java
similarity index 97%
rename from hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/GroupingHashTable.java
rename to hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/hash/GroupingHashTable.java
index 43bb0ae..801e7c7 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/GroupingHashTable.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/hash/GroupingHashTable.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.dataflow.std.group;
+package edu.uci.ics.hyracks.dataflow.std.group.hash;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -32,6 +32,9 @@
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTuplePairComparator;
+import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
class GroupingHashTable {
/**
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/hash/HashGroupBuildOperatorNodePushable.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/hash/HashGroupBuildOperatorNodePushable.java
new file mode 100644
index 0000000..0b73766
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/hash/HashGroupBuildOperatorNodePushable.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group.hash;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+
+class HashGroupBuildOperatorNodePushable extends AbstractUnaryInputSinkOperatorNodePushable {
+ private final IHyracksTaskContext ctx;
+ private final FrameTupleAccessor accessor;
+ private final Object stateId;
+ private final int[] keys;
+ private final ITuplePartitionComputerFactory tpcf;
+ private final IBinaryComparatorFactory[] comparatorFactories;
+ private final IAggregatorDescriptorFactory aggregatorFactory;
+ private final int tableSize;
+ private final RecordDescriptor inRecordDescriptor;
+ private final RecordDescriptor outRecordDescriptor;
+
+ private HashGroupState state;
+
+ HashGroupBuildOperatorNodePushable(IHyracksTaskContext ctx, Object stateId, int[] keys,
+ ITuplePartitionComputerFactory tpcf, IBinaryComparatorFactory[] comparatorFactories,
+ IAggregatorDescriptorFactory aggregatorFactory, int tableSize, RecordDescriptor inRecordDescriptor,
+ RecordDescriptor outRecordDescriptor) {
+ this.ctx = ctx;
+ this.accessor = new FrameTupleAccessor(ctx.getFrameSize(), inRecordDescriptor);
+ this.stateId = stateId;
+ this.keys = keys;
+ this.tpcf = tpcf;
+ this.comparatorFactories = comparatorFactories;
+ this.aggregatorFactory = aggregatorFactory;
+ this.tableSize = tableSize;
+ this.inRecordDescriptor = inRecordDescriptor;
+ this.outRecordDescriptor = outRecordDescriptor;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ state = new HashGroupState(ctx.getJobletContext().getJobId(), stateId);
+ state.setHashTable(new GroupingHashTable(ctx, keys, comparatorFactories, tpcf, aggregatorFactory,
+ inRecordDescriptor, outRecordDescriptor, tableSize));
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ GroupingHashTable table = state.getHashTable();
+ accessor.reset(buffer);
+ int tupleCount = accessor.getTupleCount();
+ for (int i = 0; i < tupleCount; ++i) {
+ try {
+ table.insert(accessor, i);
+ } catch (Exception e) {
+ System.out.println(e.toString());
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ ctx.setStateObject(state);
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ throw new HyracksDataException("HashGroupOperator is failed.");
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/hash/HashGroupOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/hash/HashGroupOperatorDescriptor.java
new file mode 100644
index 0000000..034e13f
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/hash/HashGroupOperatorDescriptor.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group.hash;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.ActivityId;
+import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.TaskId;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+
+/**
+ *
+ */
+public class HashGroupOperatorDescriptor extends AbstractOperatorDescriptor {
+
+ private static final int HASH_BUILD_ACTIVITY_ID = 0;
+
+ private static final int OUTPUT_ACTIVITY_ID = 1;
+
+ private static final long serialVersionUID = 1L;
+
+ private final int[] keys;
+ private final ITuplePartitionComputerFactory tpcf;
+ private final IBinaryComparatorFactory[] comparatorFactories;
+
+ private final IAggregatorDescriptorFactory aggregatorFactory;
+
+ private final int tableSize;
+
+ public HashGroupOperatorDescriptor(IOperatorDescriptorRegistry spec, int[] keys,
+ ITuplePartitionComputerFactory tpcf, IBinaryComparatorFactory[] comparatorFactories,
+ IAggregatorDescriptorFactory aggregatorFactory, RecordDescriptor outRecordDescriptor, int tableSize) {
+ super(spec, 1, 1);
+ this.keys = keys;
+ this.tpcf = tpcf;
+ this.comparatorFactories = comparatorFactories;
+ this.aggregatorFactory = aggregatorFactory;
+ recordDescriptors[0] = outRecordDescriptor;
+ this.tableSize = tableSize;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor#contributeActivities
+ * (edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder)
+ */
+ @Override
+ public void contributeActivities(IActivityGraphBuilder builder) {
+ HashBuildActivity ha = new HashBuildActivity(new ActivityId(odId, HASH_BUILD_ACTIVITY_ID));
+ builder.addActivity(this, ha);
+
+ OutputActivity oa = new OutputActivity(new ActivityId(odId, OUTPUT_ACTIVITY_ID));
+ builder.addActivity(this, oa);
+
+ builder.addSourceEdge(0, ha, 0);
+ builder.addTargetEdge(0, oa, 0);
+ builder.addBlockingEdge(ha, oa);
+ }
+
+ private class HashBuildActivity extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ public HashBuildActivity(ActivityId id) {
+ super(id);
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ final IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
+ return new HashGroupBuildOperatorNodePushable(ctx, new TaskId(getActivityId(), partition), keys, tpcf,
+ comparatorFactories, aggregatorFactory, tableSize, recordDescProvider.getInputRecordDescriptor(
+ getActivityId(), 0), recordDescriptors[0]);
+ }
+ }
+
+ private class OutputActivity extends AbstractActivityNode {
+ private static final long serialVersionUID = 1L;
+
+ public OutputActivity(ActivityId id) {
+ super(id);
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
+ return new HashGroupOutputOperatorNodePushable(ctx, new TaskId(new ActivityId(getOperatorId(),
+ HASH_BUILD_ACTIVITY_ID), partition));
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/hash/HashGroupOutputOperatorNodePushable.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/hash/HashGroupOutputOperatorNodePushable.java
new file mode 100644
index 0000000..9e3ada0
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/hash/HashGroupOutputOperatorNodePushable.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group.hash;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
+
+class HashGroupOutputOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable {
+ private final IHyracksTaskContext ctx;
+ private final Object stateId;
+
+ HashGroupOutputOperatorNodePushable(IHyracksTaskContext ctx, Object stateId) {
+ this.ctx = ctx;
+ this.stateId = stateId;
+ }
+
+ @Override
+ public void initialize() throws HyracksDataException {
+ HashGroupState buildState = (HashGroupState) ctx.getStateObject(stateId);
+ GroupingHashTable table = buildState.getHashTable();
+ writer.open();
+ try {
+ table.write(writer);
+ } catch (Exception e) {
+ writer.fail();
+ throw new HyracksDataException(e);
+ } finally {
+ writer.close();
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/hash/HashGroupState.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/hash/HashGroupState.java
new file mode 100644
index 0000000..adaaaa6
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/hash/HashGroupState.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group.hash;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractStateObject;
+
+public class HashGroupState extends AbstractStateObject {
+ private GroupingHashTable table;
+
+ public HashGroupState() {
+ }
+
+ HashGroupState(JobId jobId, Object id) {
+ super(jobId, id);
+ }
+
+ public GroupingHashTable getHashTable() {
+ return table;
+ }
+
+ public void setHashTable(GroupingHashTable table) {
+ this.table = table;
+ }
+
+ @Override
+ public void toBytes(DataOutput out) throws IOException {
+
+ }
+
+ @Override
+ public void fromBytes(DataInput in) throws IOException {
+
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/preclustered/PreclusteredGroupOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/preclustered/PreclusteredGroupOperatorDescriptor.java
new file mode 100644
index 0000000..d37b818
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/preclustered/PreclusteredGroupOperatorDescriptor.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group.preclustered;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+
+public class PreclusteredGroupOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+ private final int[] groupFields;
+ private final IBinaryComparatorFactory[] comparatorFactories;
+ private final IAggregatorDescriptorFactory aggregatorFactory;
+
+ private static final long serialVersionUID = 1L;
+
+ public PreclusteredGroupOperatorDescriptor(IOperatorDescriptorRegistry spec, int[] groupFields,
+ IBinaryComparatorFactory[] comparatorFactories, IAggregatorDescriptorFactory aggregatorFactory,
+ RecordDescriptor recordDescriptor) {
+ super(spec, 1, 1);
+ this.groupFields = groupFields;
+ this.comparatorFactories = comparatorFactories;
+ this.aggregatorFactory = aggregatorFactory;
+ recordDescriptors[0] = recordDescriptor;
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ final IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions)
+ throws HyracksDataException {
+ return new PreclusteredGroupOperatorNodePushable(ctx, groupFields, comparatorFactories, aggregatorFactory,
+ recordDescProvider.getInputRecordDescriptor(getActivityId(), 0), recordDescriptors[0]);
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/preclustered/PreclusteredGroupOperatorNodePushable.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/preclustered/PreclusteredGroupOperatorNodePushable.java
new file mode 100644
index 0000000..4acda14
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/preclustered/PreclusteredGroupOperatorNodePushable.java
@@ -0,0 +1,70 @@
+package edu.uci.ics.hyracks.dataflow.std.group.preclustered;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+
+class PreclusteredGroupOperatorNodePushable extends
+ AbstractUnaryInputUnaryOutputOperatorNodePushable {
+ private final IHyracksTaskContext ctx;
+ private final int[] groupFields;
+ private final IBinaryComparatorFactory[] comparatorFactories;
+ private final IAggregatorDescriptorFactory aggregatorFactory;
+ private final RecordDescriptor inRecordDescriptor;
+ private final RecordDescriptor outRecordDescriptor;
+ private PreclusteredGroupWriter pgw;
+
+ PreclusteredGroupOperatorNodePushable(IHyracksTaskContext ctx, int[] groupFields,
+ IBinaryComparatorFactory[] comparatorFactories, IAggregatorDescriptorFactory aggregatorFactory,
+ RecordDescriptor inRecordDescriptor, RecordDescriptor outRecordDescriptor) {
+ this.ctx = ctx;
+ this.groupFields = groupFields;
+ this.comparatorFactories = comparatorFactories;
+ this.aggregatorFactory = aggregatorFactory;
+ this.inRecordDescriptor = inRecordDescriptor;
+ this.outRecordDescriptor = outRecordDescriptor;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
+ for (int i = 0; i < comparatorFactories.length; ++i) {
+ comparators[i] = comparatorFactories[i].createBinaryComparator();
+ }
+ final IAggregatorDescriptor aggregator = aggregatorFactory.createAggregator(ctx, inRecordDescriptor,
+ outRecordDescriptor, groupFields, groupFields);
+ final ByteBuffer copyFrame = ctx.allocateFrame();
+ final FrameTupleAccessor copyFrameAccessor = new FrameTupleAccessor(ctx.getFrameSize(), inRecordDescriptor);
+ copyFrameAccessor.reset(copyFrame);
+ ByteBuffer outFrame = ctx.allocateFrame();
+ final FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+ appender.reset(outFrame, true);
+ pgw = new PreclusteredGroupWriter(ctx, groupFields, comparators, aggregator, inRecordDescriptor,
+ outRecordDescriptor, writer);
+ pgw.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ pgw.nextFrame(buffer);
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ pgw.fail();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ pgw.close();
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/PreclusteredGroupWriter.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/preclustered/PreclusteredGroupWriter.java
similarity index 96%
rename from hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/PreclusteredGroupWriter.java
rename to hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/preclustered/PreclusteredGroupWriter.java
index 89955c0..d1fec29 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/PreclusteredGroupWriter.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/preclustered/PreclusteredGroupWriter.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.dataflow.std.group;
+package edu.uci.ics.hyracks.dataflow.std.group.preclustered;
import java.nio.ByteBuffer;
@@ -25,6 +25,8 @@
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptor;
public class PreclusteredGroupWriter implements IFrameWriter {
private final int[] groupFields;
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinOperatorDescriptor.java
index 773e1e5..6c58d6f 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinOperatorDescriptor.java
@@ -14,43 +14,19 @@
*/
package edu.uci.ics.hyracks.dataflow.std.join;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.ActivityId;
import edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder;
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.TaskId;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.INullWriter;
import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTuplePairComparator;
-import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
-import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.partition.RepartitionComputerFactory;
-import edu.uci.ics.hyracks.dataflow.common.io.RunFileReader;
-import edu.uci.ics.hyracks.dataflow.common.io.RunFileWriter;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractTaskState;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
-import edu.uci.ics.hyracks.dataflow.std.structures.ISerializableTable;
-import edu.uci.ics.hyracks.dataflow.std.structures.SerializableHashTable;
public class GraceHashJoinOperatorDescriptor extends AbstractOperatorDescriptor {
private static final int RPARTITION_ACTIVITY_ID = 0;
@@ -69,9 +45,10 @@
private final boolean isLeftOuter;
private final INullWriterFactory[] nullWriterFactories1;
- public GraceHashJoinOperatorDescriptor(JobSpecification spec, int memsize, int inputsize0, int recordsPerFrame,
- double factor, int[] keys0, int[] keys1, IBinaryHashFunctionFactory[] hashFunctionFactories,
- IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDescriptor) {
+ public GraceHashJoinOperatorDescriptor(IOperatorDescriptorRegistry spec, int memsize, int inputsize0,
+ int recordsPerFrame, double factor, int[] keys0, int[] keys1,
+ IBinaryHashFunctionFactory[] hashFunctionFactories, IBinaryComparatorFactory[] comparatorFactories,
+ RecordDescriptor recordDescriptor) {
super(spec, 2, 1);
this.memsize = memsize;
this.inputsize0 = inputsize0;
@@ -86,10 +63,10 @@
recordDescriptors[0] = recordDescriptor;
}
- public GraceHashJoinOperatorDescriptor(JobSpecification spec, int memsize, int inputsize0, int recordsPerFrame,
- double factor, int[] keys0, int[] keys1, IBinaryHashFunctionFactory[] hashFunctionFactories,
- IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDescriptor, boolean isLeftOuter,
- INullWriterFactory[] nullWriterFactories1) {
+ public GraceHashJoinOperatorDescriptor(IOperatorDescriptorRegistry spec, int memsize, int inputsize0,
+ int recordsPerFrame, double factor, int[] keys0, int[] keys1,
+ IBinaryHashFunctionFactory[] hashFunctionFactories, IBinaryComparatorFactory[] comparatorFactories,
+ RecordDescriptor recordDescriptor, boolean isLeftOuter, INullWriterFactory[] nullWriterFactories1) {
super(spec, 2, 1);
this.memsize = memsize;
this.inputsize0 = inputsize0;
@@ -106,19 +83,19 @@
@Override
public void contributeActivities(IActivityGraphBuilder builder) {
- HashPartitionActivityNode rpart = new HashPartitionActivityNode(new ActivityId(odId, RPARTITION_ACTIVITY_ID),
- keys0, 0);
- HashPartitionActivityNode spart = new HashPartitionActivityNode(new ActivityId(odId, SPARTITION_ACTIVITY_ID),
- keys1, 1);
- JoinActivityNode join = new JoinActivityNode(new ActivityId(odId, JOIN_ACTIVITY_ID));
+ ActivityId rpartAid = new ActivityId(odId, RPARTITION_ACTIVITY_ID);
+ HashPartitionActivityNode rpart = new HashPartitionActivityNode(rpartAid, keys0);
+ ActivityId spartAid = new ActivityId(odId, SPARTITION_ACTIVITY_ID);
+ HashPartitionActivityNode spart = new HashPartitionActivityNode(spartAid, keys1);
+ JoinActivityNode join = new JoinActivityNode(new ActivityId(odId, JOIN_ACTIVITY_ID), rpartAid, spartAid);
- builder.addActivity(rpart);
+ builder.addActivity(this, rpart);
builder.addSourceEdge(0, rpart, 0);
- builder.addActivity(spart);
+ builder.addActivity(this, spart);
builder.addSourceEdge(1, spart, 0);
- builder.addActivity(join);
+ builder.addActivity(this, join);
builder.addBlockingEdge(rpart, spart);
builder.addBlockingEdge(spart, join);
@@ -129,226 +106,49 @@
return memsize;
}
- public static class HashPartitionTaskState extends AbstractTaskState {
- private RunFileWriter[] fWriters;
-
- public HashPartitionTaskState(JobId jobId, TaskId taskId) {
- super(jobId, taskId);
- }
-
- @Override
- public void toBytes(DataOutput out) throws IOException {
-
- }
-
- @Override
- public void fromBytes(DataInput in) throws IOException {
-
- }
- }
-
private class HashPartitionActivityNode extends AbstractActivityNode {
private static final long serialVersionUID = 1L;
- private int operatorInputIndex;
private int keys[];
- public HashPartitionActivityNode(ActivityId id, int keys[], int operatorInputIndex) {
+ public HashPartitionActivityNode(ActivityId id, int keys[]) {
super(id);
this.keys = keys;
- this.operatorInputIndex = operatorInputIndex;
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
- final IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) {
- final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
- for (int i = 0; i < comparatorFactories.length; ++i) {
- comparators[i] = comparatorFactories[i].createBinaryComparator();
- }
- final int numPartitions = (int) Math.ceil(Math.sqrt(inputsize0 * factor / nPartitions));
- IOperatorNodePushable op = new AbstractUnaryInputSinkOperatorNodePushable() {
- private final FrameTupleAccessor accessor0 = new FrameTupleAccessor(ctx.getFrameSize(),
- recordDescProvider.getInputRecordDescriptor(getOperatorId(), operatorInputIndex));
-
- private final ITuplePartitionComputer hpc = new FieldHashPartitionComputerFactory(keys,
- hashFunctionFactories).createPartitioner();
-
- private final FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
-
- private ByteBuffer[] outbufs;
-
- private HashPartitionTaskState state;
-
- @Override
- public void close() throws HyracksDataException {
- for (int i = 0; i < numPartitions; i++) {
- ByteBuffer head = outbufs[i];
- accessor0.reset(head);
- if (accessor0.getTupleCount() > 0) {
- write(i, head);
- }
- closeWriter(i);
- }
-
- ctx.setTaskState(state);
- }
-
- private void closeWriter(int i) throws HyracksDataException {
- RunFileWriter writer = state.fWriters[i];
- if (writer != null) {
- writer.close();
- }
- }
-
- private void write(int i, ByteBuffer head) throws HyracksDataException {
- RunFileWriter writer = state.fWriters[i];
- if (writer == null) {
- FileReference file = ctx.getJobletContext().createManagedWorkspaceFile(
- GraceHashJoinOperatorDescriptor.class.getSimpleName());
- writer = new RunFileWriter(file, ctx.getIOManager());
- writer.open();
- state.fWriters[i] = writer;
- }
- writer.nextFrame(head);
- }
-
- @Override
- public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
- accessor0.reset(buffer);
- int tCount = accessor0.getTupleCount();
- for (int i = 0; i < tCount; ++i) {
-
- int entry = hpc.partition(accessor0, i, numPartitions);
- ByteBuffer outbuf = outbufs[entry];
- appender.reset(outbuf, false);
- if (!appender.append(accessor0, i)) {
- // buffer is full, ie. we cannot fit the tuple
- // into the buffer -- write it to disk
- write(entry, outbuf);
- outbuf.clear();
- appender.reset(outbuf, true);
- if (!appender.append(accessor0, i)) {
- throw new HyracksDataException("Item too big to fit in frame");
- }
- }
- }
- }
-
- @Override
- public void open() throws HyracksDataException {
- state = new HashPartitionTaskState(ctx.getJobletContext().getJobId(), new TaskId(getActivityId(),
- partition));
- outbufs = new ByteBuffer[numPartitions];
- state.fWriters = new RunFileWriter[numPartitions];
- for (int i = 0; i < numPartitions; i++) {
- outbufs[i] = ctx.allocateFrame();
- }
- }
-
- @Override
- public void fail() throws HyracksDataException {
- }
- };
- return op;
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ return new GraceHashJoinPartitionBuildOperatorNodePushable(ctx, new TaskId(getActivityId(), partition),
+ keys, hashFunctionFactories, comparatorFactories, (int) Math.ceil(Math.sqrt(inputsize0 * factor
+ / nPartitions)), recordDescProvider.getInputRecordDescriptor(getActivityId(), 0));
}
}
private class JoinActivityNode extends AbstractActivityNode {
private static final long serialVersionUID = 1L;
- public JoinActivityNode(ActivityId id) {
+ private final ActivityId rpartAid;
+
+ private final ActivityId spartAid;
+
+ public JoinActivityNode(ActivityId id, ActivityId rpartAid, ActivityId spartAid) {
super(id);
+ this.rpartAid = rpartAid;
+ this.spartAid = spartAid;
}
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
final IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) {
- final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
- for (int i = 0; i < comparatorFactories.length; ++i) {
- comparators[i] = comparatorFactories[i].createBinaryComparator();
- }
- final RecordDescriptor rd0 = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0);
- final RecordDescriptor rd1 = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 1);
- final INullWriter[] nullWriters1 = isLeftOuter ? new INullWriter[nullWriterFactories1.length] : null;
- if (isLeftOuter) {
- for (int i = 0; i < nullWriterFactories1.length; i++) {
- nullWriters1[i] = nullWriterFactories1[i].createNullWriter();
- }
- }
+ final RecordDescriptor rd0 = recordDescProvider.getInputRecordDescriptor(rpartAid, 0);
+ final RecordDescriptor rd1 = recordDescProvider.getInputRecordDescriptor(spartAid, 0);
+ int numPartitions = (int) Math.ceil(Math.sqrt(inputsize0 * factor / nPartitions));
- IOperatorNodePushable op = new AbstractUnaryOutputSourceOperatorNodePushable() {
- private InMemoryHashJoin joiner;
-
- private RunFileWriter[] buildWriters;
- private RunFileWriter[] probeWriters;
- private final int numPartitions = (int) Math.ceil(Math.sqrt(inputsize0 * factor / nPartitions));
-
- @Override
- public void initialize() throws HyracksDataException {
- HashPartitionTaskState rState = (HashPartitionTaskState) ctx.getTaskState(new TaskId(
- new ActivityId(getOperatorId(), RPARTITION_ACTIVITY_ID), partition));
- HashPartitionTaskState sState = (HashPartitionTaskState) ctx.getTaskState(new TaskId(
- new ActivityId(getOperatorId(), SPARTITION_ACTIVITY_ID), partition));
- buildWriters = sState.fWriters;
- probeWriters = rState.fWriters;
-
- ITuplePartitionComputer hpcRep0 = new RepartitionComputerFactory(numPartitions,
- new FieldHashPartitionComputerFactory(keys0, hashFunctionFactories)).createPartitioner();
- ITuplePartitionComputer hpcRep1 = new RepartitionComputerFactory(numPartitions,
- new FieldHashPartitionComputerFactory(keys1, hashFunctionFactories)).createPartitioner();
-
- writer.open();// open for probe
-
- try {
-
- ByteBuffer buffer = ctx.allocateFrame();// input
- // buffer
- int tableSize = (int) (numPartitions * recordsPerFrame * factor);
- ISerializableTable table = new SerializableHashTable(tableSize, ctx);
-
- for (int partitionid = 0; partitionid < numPartitions; partitionid++) {
- RunFileWriter buildWriter = buildWriters[partitionid];
- RunFileWriter probeWriter = probeWriters[partitionid];
- if ((buildWriter == null && !isLeftOuter) || probeWriter == null) {
- continue;
- }
- table.reset();
- joiner = new InMemoryHashJoin(ctx, tableSize, new FrameTupleAccessor(ctx.getFrameSize(),
- rd0), hpcRep0, new FrameTupleAccessor(ctx.getFrameSize(), rd1), hpcRep1,
- new FrameTuplePairComparator(keys0, keys1, comparators), isLeftOuter, nullWriters1, table);
-
- // build
- if (buildWriter != null) {
- RunFileReader buildReader = buildWriter.createReader();
- buildReader.open();
- while (buildReader.nextFrame(buffer)) {
- ByteBuffer copyBuffer = ctx.allocateFrame();
- FrameUtils.copy(buffer, copyBuffer);
- joiner.build(copyBuffer);
- buffer.clear();
- }
- buildReader.close();
- }
-
- // probe
- RunFileReader probeReader = probeWriter.createReader();
- probeReader.open();
- while (probeReader.nextFrame(buffer)) {
- joiner.join(buffer, writer);
- buffer.clear();
- }
- probeReader.close();
- joiner.closeJoin(writer);
- }
- } catch (Exception e) {
- writer.fail();
- throw new HyracksDataException(e);
- } finally {
- writer.close();
- }
- }
- };
- return op;
+ return new GraceHashJoinOperatorNodePushable(ctx, new TaskId(new ActivityId(getOperatorId(),
+ RPARTITION_ACTIVITY_ID), partition), new TaskId(new ActivityId(getOperatorId(),
+ SPARTITION_ACTIVITY_ID), partition), recordsPerFrame, factor, keys0, keys1, hashFunctionFactories,
+ comparatorFactories, nullWriterFactories1, rd1, rd0, recordDescriptors[0], numPartitions,
+ isLeftOuter);
}
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinOperatorNodePushable.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinOperatorNodePushable.java
new file mode 100644
index 0000000..91f509d
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinOperatorNodePushable.java
@@ -0,0 +1,149 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.join;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriter;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTuplePairComparator;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.RepartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.common.io.RunFileReader;
+import edu.uci.ics.hyracks.dataflow.common.io.RunFileWriter;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
+import edu.uci.ics.hyracks.dataflow.std.structures.ISerializableTable;
+import edu.uci.ics.hyracks.dataflow.std.structures.SerializableHashTable;
+
+class GraceHashJoinOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable {
+ private final IHyracksTaskContext ctx;
+ private final Object state0Id;
+ private final Object state1Id;
+ private final int[] keys0;
+ private final int[] keys1;
+ private final IBinaryHashFunctionFactory[] hashFunctionFactories;
+ private final IBinaryComparatorFactory[] comparatorFactories;
+ private final INullWriterFactory[] nullWriterFactories;
+ private final RecordDescriptor rd0;
+ private final RecordDescriptor rd1;
+ private final int recordsPerFrame;
+ private final double factor;
+ private final int numPartitions;
+ private final boolean isLeftOuter;
+
+ GraceHashJoinOperatorNodePushable(IHyracksTaskContext ctx, Object state0Id, Object state1Id, int recordsPerFrame,
+ double factor, int[] keys0, int[] keys1, IBinaryHashFunctionFactory[] hashFunctionFactories,
+ IBinaryComparatorFactory[] comparatorFactories, INullWriterFactory[] nullWriterFactories,
+ RecordDescriptor rd1, RecordDescriptor rd0, RecordDescriptor outRecordDescriptor, int numPartitions,
+ boolean isLeftOuter) {
+ this.ctx = ctx;
+ this.state0Id = state0Id;
+ this.state1Id = state1Id;
+ this.keys0 = keys0;
+ this.keys1 = keys1;
+ this.hashFunctionFactories = hashFunctionFactories;
+ this.comparatorFactories = comparatorFactories;
+ this.nullWriterFactories = nullWriterFactories;
+ this.rd0 = rd0;
+ this.rd1 = rd1;
+ this.numPartitions = numPartitions;
+ this.recordsPerFrame = recordsPerFrame;
+ this.factor = factor;
+ this.isLeftOuter = isLeftOuter;
+ }
+
+ @Override
+ public void initialize() throws HyracksDataException {
+ GraceHashJoinPartitionState rState = (GraceHashJoinPartitionState) ctx.getStateObject(state0Id);
+ GraceHashJoinPartitionState sState = (GraceHashJoinPartitionState) ctx.getStateObject(state1Id);
+ RunFileWriter[] buildWriters = sState.getRunWriters();
+ RunFileWriter[] probeWriters = rState.getRunWriters();
+
+ IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
+ for (int i = 0; i < comparatorFactories.length; ++i) {
+ comparators[i] = comparatorFactories[i].createBinaryComparator();
+ }
+ ITuplePartitionComputer hpcRep0 = new RepartitionComputerFactory(numPartitions,
+ new FieldHashPartitionComputerFactory(keys0, hashFunctionFactories)).createPartitioner();
+ ITuplePartitionComputer hpcRep1 = new RepartitionComputerFactory(numPartitions,
+ new FieldHashPartitionComputerFactory(keys1, hashFunctionFactories)).createPartitioner();
+
+ final INullWriter[] nullWriters1 = isLeftOuter ? new INullWriter[nullWriterFactories.length] : null;
+ if (isLeftOuter) {
+ for (int i = 0; i < nullWriterFactories.length; i++) {
+ nullWriters1[i] = nullWriterFactories[i].createNullWriter();
+ }
+ }
+
+ writer.open();// open for probe
+
+ try {
+
+ ByteBuffer buffer = ctx.allocateFrame();// input
+ // buffer
+ int tableSize = (int) (numPartitions * recordsPerFrame * factor);
+ ISerializableTable table = new SerializableHashTable(tableSize, ctx);
+
+ for (int partitionid = 0; partitionid < numPartitions; partitionid++) {
+ RunFileWriter buildWriter = buildWriters[partitionid];
+ RunFileWriter probeWriter = probeWriters[partitionid];
+ if ((buildWriter == null && !isLeftOuter) || probeWriter == null) {
+ continue;
+ }
+ table.reset();
+ InMemoryHashJoin joiner = new InMemoryHashJoin(ctx, tableSize, new FrameTupleAccessor(
+ ctx.getFrameSize(), rd0), hpcRep0, new FrameTupleAccessor(ctx.getFrameSize(), rd1), hpcRep1,
+ new FrameTuplePairComparator(keys0, keys1, comparators), isLeftOuter, nullWriters1, table);
+
+ // build
+ if (buildWriter != null) {
+ RunFileReader buildReader = buildWriter.createReader();
+ buildReader.open();
+ while (buildReader.nextFrame(buffer)) {
+ ByteBuffer copyBuffer = ctx.allocateFrame();
+ FrameUtils.copy(buffer, copyBuffer);
+ joiner.build(copyBuffer);
+ buffer.clear();
+ }
+ buildReader.close();
+ }
+
+ // probe
+ RunFileReader probeReader = probeWriter.createReader();
+ probeReader.open();
+ while (probeReader.nextFrame(buffer)) {
+ joiner.join(buffer, writer);
+ buffer.clear();
+ }
+ probeReader.close();
+ joiner.closeJoin(writer);
+ }
+ } catch (Exception e) {
+ writer.fail();
+ throw new HyracksDataException(e);
+ } finally {
+ writer.close();
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinPartitionBuildOperatorNodePushable.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinPartitionBuildOperatorNodePushable.java
new file mode 100644
index 0000000..719b736
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinPartitionBuildOperatorNodePushable.java
@@ -0,0 +1,128 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.join;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.common.io.RunFileWriter;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
+
+class GraceHashJoinPartitionBuildOperatorNodePushable extends
+ AbstractUnaryInputSinkOperatorNodePushable {
+ private final IHyracksTaskContext ctx;
+ private final Object stateId;
+ private final int numPartitions;
+ private final IBinaryComparator[] comparators;
+ private final FrameTupleAccessor accessor0;
+ private final ITuplePartitionComputer hpc;
+ private final FrameTupleAppender appender;
+ private ByteBuffer[] outbufs;
+ private GraceHashJoinPartitionState state;
+
+ GraceHashJoinPartitionBuildOperatorNodePushable(IHyracksTaskContext ctx, Object stateId, int[] keys,
+ IBinaryHashFunctionFactory[] hashFunctionFactories, IBinaryComparatorFactory[] comparatorFactories,
+ int numPartitions, RecordDescriptor inRecordDescriptor) {
+ this.ctx = ctx;
+ this.stateId = stateId;
+ this.numPartitions = numPartitions;
+ accessor0 = new FrameTupleAccessor(ctx.getFrameSize(), inRecordDescriptor);
+ appender = new FrameTupleAppender(ctx.getFrameSize());
+ hpc = new FieldHashPartitionComputerFactory(keys, hashFunctionFactories).createPartitioner();
+ comparators = new IBinaryComparator[comparatorFactories.length];
+ for (int i = 0; i < comparatorFactories.length; ++i) {
+ comparators[i] = comparatorFactories[i].createBinaryComparator();
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ for (int i = 0; i < numPartitions; i++) {
+ ByteBuffer head = outbufs[i];
+ accessor0.reset(head);
+ if (accessor0.getTupleCount() > 0) {
+ write(i, head);
+ }
+ closeWriter(i);
+ }
+
+ ctx.setStateObject(state);
+ }
+
+ private void closeWriter(int i) throws HyracksDataException {
+ RunFileWriter writer = state.getRunWriters()[i];
+ if (writer != null) {
+ writer.close();
+ }
+ }
+
+ private void write(int i, ByteBuffer head) throws HyracksDataException {
+ RunFileWriter writer = state.getRunWriters()[i];
+ if (writer == null) {
+ FileReference file = ctx.getJobletContext().createManagedWorkspaceFile(
+ GraceHashJoinOperatorDescriptor.class.getSimpleName());
+ writer = new RunFileWriter(file, ctx.getIOManager());
+ writer.open();
+ state.getRunWriters()[i] = writer;
+ }
+ writer.nextFrame(head);
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ accessor0.reset(buffer);
+ int tCount = accessor0.getTupleCount();
+ for (int i = 0; i < tCount; ++i) {
+
+ int entry = hpc.partition(accessor0, i, numPartitions);
+ ByteBuffer outbuf = outbufs[entry];
+ appender.reset(outbuf, false);
+ if (!appender.append(accessor0, i)) {
+ // buffer is full, ie. we cannot fit the tuple
+ // into the buffer -- write it to disk
+ write(entry, outbuf);
+ outbuf.clear();
+ appender.reset(outbuf, true);
+ if (!appender.append(accessor0, i)) {
+ throw new HyracksDataException("Item too big to fit in frame");
+ }
+ }
+ }
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ state = new GraceHashJoinPartitionState(ctx.getJobletContext().getJobId(), stateId);
+ outbufs = new ByteBuffer[numPartitions];
+ state.setRunWriters(new RunFileWriter[numPartitions]);
+ for (int i = 0; i < numPartitions; i++) {
+ outbufs[i] = ctx.allocateFrame();
+ }
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinPartitionState.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinPartitionState.java
new file mode 100644
index 0000000..906042e
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinPartitionState.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.join;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.dataflow.common.io.RunFileWriter;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractStateObject;
+
+public class GraceHashJoinPartitionState extends AbstractStateObject {
+ private RunFileWriter[] fWriters;
+
+ public GraceHashJoinPartitionState(JobId jobId, Object id) {
+ super(jobId, id);
+ }
+
+ public RunFileWriter[] getRunWriters() {
+ return fWriters;
+ }
+
+ public void setRunWriters(RunFileWriter[] fWriters) {
+ this.fWriters = fWriters;
+ }
+
+ @Override
+ public void toBytes(DataOutput out) throws IOException {
+
+ }
+
+ @Override
+ public void fromBytes(DataInput in) throws IOException {
+
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/HybridHashJoinOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/HybridHashJoinOperatorDescriptor.java
index 99f0020..4f9b987 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/HybridHashJoinOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/HybridHashJoinOperatorDescriptor.java
@@ -35,8 +35,8 @@
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTuplePairComparator;
@@ -47,7 +47,7 @@
import edu.uci.ics.hyracks.dataflow.common.io.RunFileWriter;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractTaskState;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractStateObject;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
import edu.uci.ics.hyracks.dataflow.std.structures.ISerializableTable;
@@ -84,10 +84,10 @@
* @param recordDescriptor
* @throws HyracksDataException
*/
- public HybridHashJoinOperatorDescriptor(JobSpecification spec, int memsize, int inputsize0, int recordsPerFrame,
- double factor, int[] keys0, int[] keys1, IBinaryHashFunctionFactory[] hashFunctionFactories,
- IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDescriptor)
- throws HyracksDataException {
+ public HybridHashJoinOperatorDescriptor(IOperatorDescriptorRegistry spec, int memsize, int inputsize0,
+ int recordsPerFrame, double factor, int[] keys0, int[] keys1,
+ IBinaryHashFunctionFactory[] hashFunctionFactories, IBinaryComparatorFactory[] comparatorFactories,
+ RecordDescriptor recordDescriptor) throws HyracksDataException {
super(spec, 2, 1);
this.memsize = memsize;
this.inputsize0 = inputsize0;
@@ -102,10 +102,11 @@
recordDescriptors[0] = recordDescriptor;
}
- public HybridHashJoinOperatorDescriptor(JobSpecification spec, int memsize, int inputsize0, int recordsPerFrame,
- double factor, int[] keys0, int[] keys1, IBinaryHashFunctionFactory[] hashFunctionFactories,
- IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDescriptor, boolean isLeftOuter,
- INullWriterFactory[] nullWriterFactories1) throws HyracksDataException {
+ public HybridHashJoinOperatorDescriptor(IOperatorDescriptorRegistry spec, int memsize, int inputsize0,
+ int recordsPerFrame, double factor, int[] keys0, int[] keys1,
+ IBinaryHashFunctionFactory[] hashFunctionFactories, IBinaryComparatorFactory[] comparatorFactories,
+ RecordDescriptor recordDescriptor, boolean isLeftOuter, INullWriterFactory[] nullWriterFactories1)
+ throws HyracksDataException {
super(spec, 2, 1);
this.memsize = memsize;
this.inputsize0 = inputsize0;
@@ -122,15 +123,15 @@
@Override
public void contributeActivities(IActivityGraphBuilder builder) {
- BuildAndPartitionActivityNode phase1 = new BuildAndPartitionActivityNode(new ActivityId(odId,
- BUILD_AND_PARTITION_ACTIVITY_ID));
- PartitionAndJoinActivityNode phase2 = new PartitionAndJoinActivityNode(new ActivityId(odId,
- PARTITION_AND_JOIN_ACTIVITY_ID));
+ ActivityId p1Aid = new ActivityId(odId, BUILD_AND_PARTITION_ACTIVITY_ID);
+ ActivityId p2Aid = new ActivityId(odId, PARTITION_AND_JOIN_ACTIVITY_ID);
+ BuildAndPartitionActivityNode phase1 = new BuildAndPartitionActivityNode(p1Aid, p2Aid);
+ PartitionAndJoinActivityNode phase2 = new PartitionAndJoinActivityNode(p2Aid, p1Aid);
- builder.addActivity(phase1);
+ builder.addActivity(this, phase1);
builder.addSourceEdge(1, phase1, 0);
- builder.addActivity(phase2);
+ builder.addActivity(this, phase2);
builder.addSourceEdge(0, phase2, 0);
builder.addBlockingEdge(phase1, phase2);
@@ -138,7 +139,7 @@
builder.addTargetEdge(0, phase2, 0);
}
- public static class BuildAndPartitionTaskState extends AbstractTaskState {
+ public static class BuildAndPartitionTaskState extends AbstractStateObject {
private RunFileWriter[] fWriters;
private InMemoryHashJoin joiner;
private int nPartitions;
@@ -166,15 +167,18 @@
private class BuildAndPartitionActivityNode extends AbstractActivityNode {
private static final long serialVersionUID = 1L;
- public BuildAndPartitionActivityNode(ActivityId id) {
+ private final ActivityId joinAid;
+
+ public BuildAndPartitionActivityNode(ActivityId id, ActivityId joinAid) {
super(id);
+ this.joinAid = joinAid;
}
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) {
- final RecordDescriptor rd0 = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0);
- final RecordDescriptor rd1 = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 1);
+ final RecordDescriptor rd0 = recordDescProvider.getInputRecordDescriptor(joinAid, 0);
+ final RecordDescriptor rd1 = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);
final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
for (int i = 0; i < comparatorFactories.length; ++i) {
comparators[i] = comparatorFactories[i].createBinaryComparator();
@@ -211,7 +215,7 @@
closeWriter(i);
}
- ctx.setTaskState(state);
+ ctx.setStateObject(state);
}
@Override
@@ -351,15 +355,18 @@
private class PartitionAndJoinActivityNode extends AbstractActivityNode {
private static final long serialVersionUID = 1L;
- public PartitionAndJoinActivityNode(ActivityId id) {
+ private final ActivityId buildAid;
+
+ public PartitionAndJoinActivityNode(ActivityId id, ActivityId buildAid) {
super(id);
+ this.buildAid = buildAid;
}
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) {
- final RecordDescriptor rd0 = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0);
- final RecordDescriptor rd1 = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 1);
+ final RecordDescriptor rd0 = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);
+ final RecordDescriptor rd1 = recordDescProvider.getInputRecordDescriptor(buildAid, 0);
final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
for (int i = 0; i < comparatorFactories.length; ++i) {
comparators[i] = comparatorFactories[i].createBinaryComparator();
@@ -390,7 +397,7 @@
@Override
public void open() throws HyracksDataException {
- state = (BuildAndPartitionTaskState) ctx.getTaskState(new TaskId(new ActivityId(getOperatorId(),
+ state = (BuildAndPartitionTaskState) ctx.getStateObject(new TaskId(new ActivityId(getOperatorId(),
BUILD_AND_PARTITION_ACTIVITY_ID), partition));
writer.open();
buildWriters = state.fWriters;
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/InMemoryHashJoinOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/InMemoryHashJoinOperatorDescriptor.java
index 61fc0b3..e0a5613 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/InMemoryHashJoinOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/InMemoryHashJoinOperatorDescriptor.java
@@ -33,24 +33,21 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTuplePairComparator;
import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractTaskState;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractStateObject;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
import edu.uci.ics.hyracks.dataflow.std.structures.ISerializableTable;
import edu.uci.ics.hyracks.dataflow.std.structures.SerializableHashTable;
public class InMemoryHashJoinOperatorDescriptor extends AbstractOperatorDescriptor {
- private static final int BUILD_ACTIVITY_ID = 0;
- private static final int PROBE_ACTIVITY_ID = 1;
-
private static final long serialVersionUID = 1L;
private final int[] keys0;
private final int[] keys1;
@@ -60,7 +57,7 @@
private final INullWriterFactory[] nullWriterFactories1;
private final int tableSize;
- public InMemoryHashJoinOperatorDescriptor(JobSpecification spec, int[] keys0, int[] keys1,
+ public InMemoryHashJoinOperatorDescriptor(IOperatorDescriptorRegistry spec, int[] keys0, int[] keys1,
IBinaryHashFunctionFactory[] hashFunctionFactories, IBinaryComparatorFactory[] comparatorFactories,
RecordDescriptor recordDescriptor, int tableSize) {
super(spec, 2, 1);
@@ -74,7 +71,7 @@
this.tableSize = tableSize;
}
- public InMemoryHashJoinOperatorDescriptor(JobSpecification spec, int[] keys0, int[] keys1,
+ public InMemoryHashJoinOperatorDescriptor(IOperatorDescriptorRegistry spec, int[] keys0, int[] keys1,
IBinaryHashFunctionFactory[] hashFunctionFactories, IBinaryComparatorFactory[] comparatorFactories,
RecordDescriptor recordDescriptor, boolean isLeftOuter, INullWriterFactory[] nullWriterFactories1,
int tableSize) {
@@ -91,13 +88,15 @@
@Override
public void contributeActivities(IActivityGraphBuilder builder) {
- HashBuildActivityNode hba = new HashBuildActivityNode(new ActivityId(odId, 0));
- HashProbeActivityNode hpa = new HashProbeActivityNode(new ActivityId(odId, 1));
+ ActivityId hbaId = new ActivityId(odId, 0);
+ ActivityId hpaId = new ActivityId(odId, 1);
+ HashBuildActivityNode hba = new HashBuildActivityNode(hbaId, hpaId);
+ HashProbeActivityNode hpa = new HashProbeActivityNode(hpaId);
- builder.addActivity(hba);
+ builder.addActivity(this, hba);
builder.addSourceEdge(1, hba, 0);
- builder.addActivity(hpa);
+ builder.addActivity(this, hpa);
builder.addSourceEdge(0, hpa, 0);
builder.addTargetEdge(0, hpa, 0);
@@ -105,7 +104,7 @@
builder.addBlockingEdge(hba, hpa);
}
- public static class HashBuildTaskState extends AbstractTaskState {
+ public static class HashBuildTaskState extends AbstractStateObject {
private InMemoryHashJoin joiner;
public HashBuildTaskState() {
@@ -129,17 +128,18 @@
private class HashBuildActivityNode extends AbstractActivityNode {
private static final long serialVersionUID = 1L;
- public HashBuildActivityNode(ActivityId id) {
+ private final ActivityId hpaId;
+
+ public HashBuildActivityNode(ActivityId id, ActivityId hpaId) {
super(id);
+ this.hpaId = hpaId;
}
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
- final RecordDescriptor rd0 = recordDescProvider
- .getInputRecordDescriptor(getOperatorId(), BUILD_ACTIVITY_ID);
- final RecordDescriptor rd1 = recordDescProvider
- .getInputRecordDescriptor(getOperatorId(), PROBE_ACTIVITY_ID);
+ final RecordDescriptor rd0 = recordDescProvider.getInputRecordDescriptor(hpaId, 0);
+ final RecordDescriptor rd1 = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);
final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
for (int i = 0; i < comparatorFactories.length; ++i) {
comparators[i] = comparatorFactories[i].createBinaryComparator();
@@ -178,7 +178,7 @@
@Override
public void close() throws HyracksDataException {
- ctx.setTaskState(state);
+ ctx.setStateObject(state);
}
@Override
@@ -204,8 +204,8 @@
@Override
public void open() throws HyracksDataException {
- state = (HashBuildTaskState) ctx.getTaskState(new TaskId(new ActivityId(getOperatorId(),
- BUILD_ACTIVITY_ID), partition));
+ state = (HashBuildTaskState) ctx.getStateObject(new TaskId(new ActivityId(getOperatorId(), 0),
+ partition));
writer.open();
}
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/JoinComparatorFactory.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/JoinComparatorFactory.java
index 6e120f0..377e9e0 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/JoinComparatorFactory.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/JoinComparatorFactory.java
@@ -1,10 +1,10 @@
package edu.uci.ics.hyracks.dataflow.std.join;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparator;
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparatorFactory;
-
public class JoinComparatorFactory implements ITuplePairComparatorFactory {
private static final long serialVersionUID = 1L;
@@ -19,7 +19,7 @@
}
@Override
- public ITuplePairComparator createTuplePairComparator() {
+ public ITuplePairComparator createTuplePairComparator(IHyracksTaskContext ctx) {
return new JoinComparator(bFactory.createBinaryComparator(), pos0, pos1);
}
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/NestedLoopJoinOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/NestedLoopJoinOperatorDescriptor.java
index 21a828c..a699703 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/NestedLoopJoinOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/NestedLoopJoinOperatorDescriptor.java
@@ -30,13 +30,13 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractTaskState;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractStateObject;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
@@ -48,8 +48,8 @@
private final ITuplePairComparatorFactory comparatorFactory;
private final int memSize;
- public NestedLoopJoinOperatorDescriptor(JobSpecification spec, ITuplePairComparatorFactory comparatorFactory,
- RecordDescriptor recordDescriptor, int memSize) {
+ public NestedLoopJoinOperatorDescriptor(IOperatorDescriptorRegistry spec,
+ ITuplePairComparatorFactory comparatorFactory, RecordDescriptor recordDescriptor, int memSize) {
super(spec, 2, 1);
this.comparatorFactory = comparatorFactory;
this.recordDescriptors[0] = recordDescriptor;
@@ -58,21 +58,22 @@
@Override
public void contributeActivities(IActivityGraphBuilder builder) {
- JoinCacheActivityNode jc = new JoinCacheActivityNode(new ActivityId(getOperatorId(), JOIN_CACHE_ACTIVITY_ID));
- NestedLoopJoinActivityNode nlj = new NestedLoopJoinActivityNode(new ActivityId(getOperatorId(),
- NL_JOIN_ACTIVITY_ID));
+ ActivityId jcaId = new ActivityId(getOperatorId(), JOIN_CACHE_ACTIVITY_ID);
+ ActivityId nljAid = new ActivityId(getOperatorId(), NL_JOIN_ACTIVITY_ID);
+ JoinCacheActivityNode jc = new JoinCacheActivityNode(jcaId, nljAid);
+ NestedLoopJoinActivityNode nlj = new NestedLoopJoinActivityNode(nljAid);
- builder.addActivity(jc);
+ builder.addActivity(this, jc);
builder.addSourceEdge(1, jc, 0);
- builder.addActivity(nlj);
+ builder.addActivity(this, nlj);
builder.addSourceEdge(0, nlj, 0);
builder.addTargetEdge(0, nlj, 0);
builder.addBlockingEdge(jc, nlj);
}
- public static class JoinCacheTaskState extends AbstractTaskState {
+ public static class JoinCacheTaskState extends AbstractStateObject {
private NestedLoopJoin joiner;
public JoinCacheTaskState() {
@@ -96,16 +97,19 @@
private class JoinCacheActivityNode extends AbstractActivityNode {
private static final long serialVersionUID = 1L;
- public JoinCacheActivityNode(ActivityId id) {
+ private final ActivityId nljAid;
+
+ public JoinCacheActivityNode(ActivityId id, ActivityId nljAid) {
super(id);
+ this.nljAid = nljAid;
}
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
- final RecordDescriptor rd0 = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0);
- final RecordDescriptor rd1 = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 1);
- final ITuplePairComparator comparator = comparatorFactory.createTuplePairComparator();
+ final RecordDescriptor rd0 = recordDescProvider.getInputRecordDescriptor(nljAid, 0);
+ final RecordDescriptor rd1 = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);
+ final ITuplePairComparator comparator = comparatorFactory.createTuplePairComparator(ctx);
IOperatorNodePushable op = new AbstractUnaryInputSinkOperatorNodePushable() {
private JoinCacheTaskState state;
@@ -129,7 +133,7 @@
@Override
public void close() throws HyracksDataException {
state.joiner.closeCache();
- ctx.setTaskState(state);
+ ctx.setStateObject(state);
}
@Override
@@ -156,7 +160,7 @@
@Override
public void open() throws HyracksDataException {
- state = (JoinCacheTaskState) ctx.getTaskState(new TaskId(new ActivityId(getOperatorId(),
+ state = (JoinCacheTaskState) ctx.getStateObject(new TaskId(new ActivityId(getOperatorId(),
JOIN_CACHE_ACTIVITY_ID), partition));
writer.open();
}
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
index 87e2b2c..3a7ee2c 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
@@ -37,8 +37,8 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFamily;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTuplePairComparator;
import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
@@ -47,7 +47,7 @@
import edu.uci.ics.hyracks.dataflow.common.io.RunFileReader;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractTaskState;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractStateObject;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
import edu.uci.ics.hyracks.dataflow.std.structures.ISerializableTable;
@@ -63,37 +63,36 @@
* of R (in terms of buffers). HHJ (Hybrid Hash Join) has two main phases: Build and Probe, where in our implementation Probe phase
* can apply HHJ recursively, based on the value of M and size of R and S. HHJ phases proceed as follow:
* BUILD:
- * – Calculate number of partitions (Based on the size of R, fudge factor and M) [See Shapiro's paper for the detailed discussion].
- * – Initialize the build phase (one frame per partition – all partitions considered resident at first)
- * – Read tuples of R, frame by frame, and hash each tuple (based on a given hash function) to find
+ * Calculate number of partitions (Based on the size of R, fudge factor and M) [See Shapiro's paper for the detailed discussion].
+ * Initialize the build phase (one frame per partition, all partitions considered resident at first)
+ * Read tuples of R, frame by frame, and hash each tuple (based on a given hash function) to find
* its target partition and try to append it to that partition:
- * – If target partition's buffer is full, try to allocate a new buffer for it.
- * – if no free buffer is available, find the largest resident partition and spill it. Using its freed
+ * If target partition's buffer is full, try to allocate a new buffer for it.
+ * if no free buffer is available, find the largest resident partition and spill it. Using its freed
* buffers after spilling, allocate a new buffer for the target partition.
- * – Being done with R, close the build phase. (During closing we write the very last buffer of each
+ * Being done with R, close the build phase. (During closing we write the very last buffer of each
* spilled partition to the disk, and we do partition tuning, where we try to bring back as many buffers, belonging to
* spilled partitions as possible into memory, based on the free buffers - We will stop at the point where remaining free buffers is not enough
* for reloading an entire partition back into memory)
- * – Create the hash table for the resident partitions (basically we create an in-memory hash join here)
+ * Create the hash table for the resident partitions (basically we create an in-memory hash join here)
* PROBE:
- * – Initialize the probe phase on S (mainly allocate one buffer per spilled partition, and one buffer
+ * Initialize the probe phase on S (mainly allocate one buffer per spilled partition, and one buffer
* for the whole resident partitions)
- * – Read tuples of S, frame by frame and hash each tuple T to its target partition P
- * – if P is a resident partition, pass T to the in-memory hash join and generate the output record,
+ * Read tuples of S, frame by frame and hash each tuple T to its target partition P
+ * if P is a resident partition, pass T to the in-memory hash join and generate the output record,
* if any matching(s) record found
- * – if P is spilled, write T to the dedicated buffer for P (on the probe side)
- * – Once scanning of S is done, we try to join partition pairs (Ri, Si) of the spilled partitions:
- * – if any of Ri or Si is smaller than M, then we simply use an in-memory hash join to join them
- * – otherwise we apply HHJ recursively:
- * – if after applying HHJ recursively, we do not gain enough size reduction (max size of the
+ * if P is spilled, write T to the dedicated buffer for P (on the probe side)
+ * Once scanning of S is done, we try to join partition pairs (Ri, Si) of the spilled partitions:
+ * if any of Ri or Si is smaller than M, then we simply use an in-memory hash join to join them
+ * otherwise we apply HHJ recursively:
+ * if after applying HHJ recursively, we do not gain enough size reduction (max size of the
* resulting partitions were more than 80% of the initial Ri,Si size) then we switch to
* nested loop join for joining.
- * – (At each step of partition-pair joining, we consider role reversal, which means if size of Si were
+ * (At each step of partition-pair joining, we consider role reversal, which means if size of Si were
* greater than Ri, then we make sure that we switch the roles of build/probe between them)
*/
public class OptimizedHybridHashJoinOperatorDescriptor extends AbstractOperatorDescriptor {
-
private static final int BUILD_AND_PARTITION_ACTIVITY_ID = 0;
private static final int PARTITION_AND_JOIN_ACTIVITY_ID = 1;
@@ -116,8 +115,8 @@
private final boolean isLeftOuter;
private final INullWriterFactory[] nullWriterFactories1;
- public OptimizedHybridHashJoinOperatorDescriptor(JobSpecification spec, int memsize, int inputsize0, double factor,
- int[] keys0, int[] keys1, IBinaryHashFunctionFamily[] hashFunctionGeneratorFactories,
+ public OptimizedHybridHashJoinOperatorDescriptor(IOperatorDescriptorRegistry spec, int memsize, int inputsize0,
+ double factor, int[] keys0, int[] keys1, IBinaryHashFunctionFamily[] hashFunctionGeneratorFactories,
IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDescriptor,
ITuplePairComparatorFactory tupPaircomparatorFactory0,
ITuplePairComparatorFactory tupPaircomparatorFactory1, boolean isLeftOuter,
@@ -139,8 +138,8 @@
}
- public OptimizedHybridHashJoinOperatorDescriptor(JobSpecification spec, int memsize, int inputsize0, double factor,
- int[] keys0, int[] keys1, IBinaryHashFunctionFamily[] hashFunctionGeneratorFactories,
+ public OptimizedHybridHashJoinOperatorDescriptor(IOperatorDescriptorRegistry spec, int memsize, int inputsize0,
+ double factor, int[] keys0, int[] keys1, IBinaryHashFunctionFamily[] hashFunctionGeneratorFactories,
IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDescriptor,
ITuplePairComparatorFactory tupPaircomparatorFactory0, ITuplePairComparatorFactory tupPaircomparatorFactory1)
throws HyracksDataException {
@@ -162,15 +161,15 @@
@Override
public void contributeActivities(IActivityGraphBuilder builder) {
- PartitionAndBuildActivityNode phase1 = new PartitionAndBuildActivityNode(new ActivityId(odId,
- BUILD_AND_PARTITION_ACTIVITY_ID));
- ProbeAndJoinActivityNode phase2 = new ProbeAndJoinActivityNode(new ActivityId(odId,
- PARTITION_AND_JOIN_ACTIVITY_ID));
+ ActivityId buildAid = new ActivityId(odId, BUILD_AND_PARTITION_ACTIVITY_ID);
+ ActivityId probeAid = new ActivityId(odId, PARTITION_AND_JOIN_ACTIVITY_ID);
+ PartitionAndBuildActivityNode phase1 = new PartitionAndBuildActivityNode(buildAid, probeAid);
+ ProbeAndJoinActivityNode phase2 = new ProbeAndJoinActivityNode(probeAid, buildAid);
- builder.addActivity(phase1);
+ builder.addActivity(this, phase1);
builder.addSourceEdge(0, phase1, 0);
- builder.addActivity(phase2);
+ builder.addActivity(this, phase2);
builder.addSourceEdge(1, phase2, 0);
builder.addBlockingEdge(phase1, phase2);
@@ -201,7 +200,7 @@
return numberOfPartitions;
}
- public static class BuildAndPartitionTaskState extends AbstractTaskState {
+ public static class BuildAndPartitionTaskState extends AbstractStateObject {
private int memForJoin;
private int numOfPartitions;
@@ -236,16 +235,19 @@
private class PartitionAndBuildActivityNode extends AbstractActivityNode {
private static final long serialVersionUID = 1L;
- public PartitionAndBuildActivityNode(ActivityId id) {
+ private final ActivityId probeAid;
+
+ public PartitionAndBuildActivityNode(ActivityId id, ActivityId probeAid) {
super(id);
+ this.probeAid = probeAid;
}
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) {
- final RecordDescriptor probeRd = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0);
- final RecordDescriptor buildRd = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 1);
+ final RecordDescriptor probeRd = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);
+ final RecordDescriptor buildRd = recordDescProvider.getInputRecordDescriptor(probeAid, 0);
final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
for (int i = 0; i < comparatorFactories.length; i++) {
@@ -290,7 +292,7 @@
@Override
public void close() throws HyracksDataException {
state.hybridHJ.closeBuild();
- ctx.setTaskState(state);
+ ctx.setStateObject(state);
}
@Override
@@ -314,19 +316,22 @@
private static final long serialVersionUID = 1L;
- public ProbeAndJoinActivityNode(ActivityId id) {
+ private final ActivityId buildAid;
+
+ public ProbeAndJoinActivityNode(ActivityId id, ActivityId buildAid) {
super(id);
+ this.buildAid = buildAid;
}
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) {
- final RecordDescriptor probeRd = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0);
- final RecordDescriptor buildRd = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 1);
+ final RecordDescriptor probeRd = recordDescProvider.getInputRecordDescriptor(buildAid, 0);
+ final RecordDescriptor buildRd = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);
final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
- final ITuplePairComparator nljComparator0 = tuplePairComparatorFactory0.createTuplePairComparator();
- final ITuplePairComparator nljComparator1 = tuplePairComparatorFactory1.createTuplePairComparator();
+ final ITuplePairComparator nljComparator0 = tuplePairComparatorFactory0.createTuplePairComparator(ctx);
+ final ITuplePairComparator nljComparator1 = tuplePairComparatorFactory1.createTuplePairComparator(ctx);
for (int i = 0; i < comparatorFactories.length; i++) {
comparators[i] = comparatorFactories[i].createBinaryComparator();
@@ -353,7 +358,7 @@
@Override
public void open() throws HyracksDataException {
- state = (BuildAndPartitionTaskState) ctx.getTaskState(new TaskId(new ActivityId(getOperatorId(),
+ state = (BuildAndPartitionTaskState) ctx.getStateObject(new TaskId(new ActivityId(getOperatorId(),
BUILD_AND_PARTITION_ACTIVITY_ID), partition));
writer.open();
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/map/DeserializedMapperOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/map/DeserializedMapperOperatorDescriptor.java
index 84caadf..30aae7f 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/map/DeserializedMapperOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/map/DeserializedMapperOperatorDescriptor.java
@@ -20,7 +20,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.IOpenableDataWriterOperator;
import edu.uci.ics.hyracks.dataflow.std.util.DeserializedOperatorNodePushable;
@@ -64,8 +64,8 @@
private final IDeserializedMapperFactory mapperFactory;
- public DeserializedMapperOperatorDescriptor(JobSpecification spec, IDeserializedMapperFactory mapperFactory,
- RecordDescriptor recordDescriptor) {
+ public DeserializedMapperOperatorDescriptor(IOperatorDescriptorRegistry spec,
+ IDeserializedMapperFactory mapperFactory, RecordDescriptor recordDescriptor) {
super(spec, 1, 1);
this.mapperFactory = mapperFactory;
recordDescriptors[0] = recordDescriptor;
@@ -75,6 +75,6 @@
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
return new DeserializedOperatorNodePushable(ctx, new MapperOperator(),
- recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0));
+ recordDescProvider.getInputRecordDescriptor(getActivityId(), 0));
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorDescriptor.java
index 85a6912..e22db5c 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorDescriptor.java
@@ -19,7 +19,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
public class ConstantTupleSourceOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
@@ -30,7 +30,7 @@
private byte[] tupleData;
private int tupleSize;
- public ConstantTupleSourceOperatorDescriptor(JobSpecification spec, RecordDescriptor recDesc, int[] fieldSlots,
+ public ConstantTupleSourceOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc, int[] fieldSlots,
byte[] tupleData, int tupleSize) {
super(spec, 0, 1);
this.tupleData = tupleData;
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/IdentityOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/IdentityOperatorDescriptor.java
new file mode 100644
index 0000000..b58b1c2
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/IdentityOperatorDescriptor.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.misc;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
+
+public class IdentityOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+ private static final long serialVersionUID = 1L;
+
+ public IdentityOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor rDesc) {
+ super(spec, 1, 1);
+ recordDescriptors[0] = rDesc;
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ final IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions)
+ throws HyracksDataException {
+ return new AbstractUnaryInputUnaryOutputOperatorNodePushable() {
+ @Override
+ public void open() throws HyracksDataException {
+ writer.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ writer.nextFrame(buffer);
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ writer.close();
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/LimitOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/LimitOperatorDescriptor.java
index 5d1d559..2d2f074 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/LimitOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/LimitOperatorDescriptor.java
@@ -21,7 +21,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
@@ -32,7 +32,7 @@
private static final long serialVersionUID = 1L;
private final int outputLimit;
- public LimitOperatorDescriptor(JobSpecification spec, RecordDescriptor rDesc, int outputLimit) {
+ public LimitOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor rDesc, int outputLimit) {
super(spec, 1, 1);
recordDescriptors[0] = rDesc;
this.outputLimit = outputLimit;
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializingOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializingOperatorDescriptor.java
index 83160e5..ba8c00c 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializingOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializingOperatorDescriptor.java
@@ -28,13 +28,13 @@
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.io.RunFileReader;
import edu.uci.ics.hyracks.dataflow.common.io.RunFileWriter;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractTaskState;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractStateObject;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
@@ -44,7 +44,7 @@
private final static int MATERIALIZER_ACTIVITY_ID = 0;
private final static int READER_ACTIVITY_ID = 1;
- public MaterializingOperatorDescriptor(JobSpecification spec, RecordDescriptor recordDescriptor) {
+ public MaterializingOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recordDescriptor) {
super(spec, 1, 1);
recordDescriptors[0] = recordDescriptor;
}
@@ -54,16 +54,16 @@
MaterializerActivityNode ma = new MaterializerActivityNode(new ActivityId(odId, MATERIALIZER_ACTIVITY_ID));
ReaderActivityNode ra = new ReaderActivityNode(new ActivityId(odId, READER_ACTIVITY_ID));
- builder.addActivity(ma);
+ builder.addActivity(this, ma);
builder.addSourceEdge(0, ma, 0);
- builder.addActivity(ra);
+ builder.addActivity(this, ra);
builder.addTargetEdge(0, ra, 0);
builder.addBlockingEdge(ma, ra);
}
- public static class MaterializerTaskState extends AbstractTaskState {
+ public static class MaterializerTaskState extends AbstractStateObject {
private RunFileWriter out;
public MaterializerTaskState() {
@@ -115,7 +115,7 @@
@Override
public void close() throws HyracksDataException {
state.out.close();
- ctx.setTaskState(state);
+ ctx.setStateObject(state);
}
@Override
@@ -139,7 +139,7 @@
@Override
public void initialize() throws HyracksDataException {
ByteBuffer frame = ctx.allocateFrame();
- MaterializerTaskState state = (MaterializerTaskState) ctx.getTaskState(new TaskId(new ActivityId(
+ MaterializerTaskState state = (MaterializerTaskState) ctx.getStateObject(new TaskId(new ActivityId(
getOperatorId(), MATERIALIZER_ACTIVITY_ID), partition));
RunFileReader in = state.out.createReader();
writer.open();
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/NullSinkOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/NullSinkOperatorDescriptor.java
index 8e0ac79..7c32c60 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/NullSinkOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/NullSinkOperatorDescriptor.java
@@ -20,14 +20,14 @@
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
public class NullSinkOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
private static final long serialVersionUID = 1L;
- public NullSinkOperatorDescriptor(JobSpecification spec) {
+ public NullSinkOperatorDescriptor(IOperatorDescriptorRegistry spec) {
super(spec, 1, 0);
}
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/PrinterOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/PrinterOperatorDescriptor.java
index 7effbb0..47a8616 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/PrinterOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/PrinterOperatorDescriptor.java
@@ -19,7 +19,7 @@
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.IOpenableDataWriterOperator;
import edu.uci.ics.hyracks.dataflow.std.util.DeserializedOperatorNodePushable;
@@ -28,7 +28,7 @@
public class PrinterOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
private static final long serialVersionUID = 1L;
- public PrinterOperatorDescriptor(JobSpecification spec) {
+ public PrinterOperatorDescriptor(IOperatorDescriptorRegistry spec) {
super(spec, 1, 0);
}
@@ -64,6 +64,6 @@
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
return new DeserializedOperatorNodePushable(ctx, new PrinterOperator(),
- recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0));
+ recordDescProvider.getInputRecordDescriptor(getActivityId(), 0));
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java
index 6874f77..231a8d7 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java
@@ -8,7 +8,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputOperatorNodePushable;
@@ -16,7 +16,7 @@
public class SplitOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
private static final long serialVersionUID = 1L;
- public SplitOperatorDescriptor(JobSpecification spec, RecordDescriptor rDesc, int outputArity) {
+ public SplitOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor rDesc, int outputArity) {
super(spec, 1, outputArity);
for (int i = 0; i < outputArity; i++) {
recordDescriptors[i] = rDesc;
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitVectorOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitVectorOperatorDescriptor.java
index 3e99000..aad0e66 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitVectorOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitVectorOperatorDescriptor.java
@@ -28,11 +28,11 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractTaskState;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractStateObject;
import edu.uci.ics.hyracks.dataflow.std.base.IOpenableDataWriterOperator;
import edu.uci.ics.hyracks.dataflow.std.util.DeserializedOperatorNodePushable;
@@ -40,7 +40,7 @@
private static final int COLLECT_ACTIVITY_ID = 0;
private static final int SPLIT_ACTIVITY_ID = 1;
- public static class CollectTaskState extends AbstractTaskState {
+ public static class CollectTaskState extends AbstractStateObject {
private ArrayList<Object[]> buffer;
public CollectTaskState() {
@@ -91,7 +91,7 @@
@Override
public void close() throws HyracksDataException {
- ctx.setTaskState(state);
+ ctx.setStateObject(state);
}
@Override
@@ -105,7 +105,7 @@
}
};
return new DeserializedOperatorNodePushable(ctx, op, recordDescProvider.getInputRecordDescriptor(
- getOperatorId(), 0));
+ getActivityId(), 0));
}
}
@@ -134,7 +134,7 @@
@Override
public void open() throws HyracksDataException {
- state = (CollectTaskState) ctx.getTaskState(new TaskId(new ActivityId(getOperatorId(),
+ state = (CollectTaskState) ctx.getStateObject(new TaskId(new ActivityId(getOperatorId(),
COLLECT_ACTIVITY_ID), partition));
}
@@ -158,8 +158,8 @@
writer.fail();
}
};
- return new DeserializedOperatorNodePushable(ctx, op, recordDescProvider.getInputRecordDescriptor(
- getOperatorId(), 0));
+ return new DeserializedOperatorNodePushable(ctx, op, recordDescProvider.getOutputRecordDescriptor(
+ getActivityId(), 0));
}
}
@@ -167,7 +167,7 @@
private final int splits;
- public SplitVectorOperatorDescriptor(JobSpecification spec, int splits, RecordDescriptor recordDescriptor) {
+ public SplitVectorOperatorDescriptor(IOperatorDescriptorRegistry spec, int splits, RecordDescriptor recordDescriptor) {
super(spec, 1, 1);
this.splits = splits;
recordDescriptors[0] = recordDescriptor;
@@ -178,10 +178,10 @@
CollectActivity ca = new CollectActivity(new ActivityId(odId, COLLECT_ACTIVITY_ID));
SplitActivity sa = new SplitActivity(new ActivityId(odId, SPLIT_ACTIVITY_ID));
- builder.addActivity(ca);
+ builder.addActivity(this, ca);
builder.addSourceEdge(0, ca, 0);
- builder.addActivity(sa);
+ builder.addActivity(this, sa);
builder.addTargetEdge(0, sa, 0);
builder.addBlockingEdge(ca, sa);
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/BSTMemMgr.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/BSTMemMgr.java
index d07ba51..0c569f2 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/BSTMemMgr.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/BSTMemMgr.java
@@ -16,7 +16,7 @@
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.api.context.IHyracksCommonContext;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
@@ -33,7 +33,7 @@
*/
public class BSTMemMgr implements IMemoryManager {
- private final IHyracksCommonContext ctx;
+ private final IHyracksTaskContext ctx;
public static int frameSize;
private ByteBuffer[] frames;
@@ -50,7 +50,7 @@
private Slot[] parentRes;
private int lastFrame;
- public BSTMemMgr(IHyracksCommonContext ctx, int memSize) {
+ public BSTMemMgr(IHyracksTaskContext ctx, int memSize) {
this.ctx = ctx;
frameSize = ctx.getFrameSize();
convertBuffer = ByteBuffer.allocate(4);
@@ -167,7 +167,6 @@
//clean up all frames
for (int i = 0; i < frames.length; i++)
frames[i] = null;
- System.gc();
}
/**
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java
index 0effefb..be71b44 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java
@@ -32,11 +32,11 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractTaskState;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractStateObject;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
@@ -51,12 +51,12 @@
private final IBinaryComparatorFactory[] comparatorFactories;
private final int framesLimit;
- public ExternalSortOperatorDescriptor(JobSpecification spec, int framesLimit, int[] sortFields,
+ public ExternalSortOperatorDescriptor(IOperatorDescriptorRegistry spec, int framesLimit, int[] sortFields,
IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDescriptor) {
this(spec, framesLimit, sortFields, null, comparatorFactories, recordDescriptor);
}
- public ExternalSortOperatorDescriptor(JobSpecification spec, int framesLimit, int[] sortFields,
+ public ExternalSortOperatorDescriptor(IOperatorDescriptorRegistry spec, int framesLimit, int[] sortFields,
INormalizedKeyComputerFactory firstKeyNormalizerFactory, IBinaryComparatorFactory[] comparatorFactories,
RecordDescriptor recordDescriptor) {
super(spec, 1, 1);
@@ -75,16 +75,16 @@
SortActivity sa = new SortActivity(new ActivityId(odId, SORT_ACTIVITY_ID));
MergeActivity ma = new MergeActivity(new ActivityId(odId, MERGE_ACTIVITY_ID));
- builder.addActivity(sa);
+ builder.addActivity(this, sa);
builder.addSourceEdge(0, sa, 0);
- builder.addActivity(ma);
+ builder.addActivity(this, ma);
builder.addTargetEdge(0, ma, 0);
builder.addBlockingEdge(sa, ma);
}
- public static class SortTaskState extends AbstractTaskState {
+ public static class SortTaskState extends AbstractStateObject {
private List<IFrameReader> runs;
private FrameSorter frameSorter;
@@ -138,7 +138,7 @@
runGen.close();
state.runs = runGen.getRuns();
state.frameSorter = runGen.getFrameSorter();
- ctx.setTaskState(state);
+ ctx.setStateObject(state);
}
@Override
@@ -163,7 +163,7 @@
IOperatorNodePushable op = new AbstractUnaryOutputSourceOperatorNodePushable() {
@Override
public void initialize() throws HyracksDataException {
- SortTaskState state = (SortTaskState) ctx.getTaskState(new TaskId(new ActivityId(getOperatorId(),
+ SortTaskState state = (SortTaskState) ctx.getStateObject(new TaskId(new ActivityId(getOperatorId(),
SORT_ACTIVITY_ID), partition));
List<IFrameReader> runs = state.runs;
FrameSorter frameSorter = state.frameSorter;
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/ExternalSortRunMerger.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/ExternalSortRunMerger.java
index 5e413da..b51132e 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/ExternalSortRunMerger.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/ExternalSortRunMerger.java
@@ -100,12 +100,9 @@
}
/** recycle sort buffer */
frameSorter.close();
- System.gc();
-
} else {
/** recycle sort buffer */
frameSorter.close();
- System.gc();
inFrames = new ArrayList<ByteBuffer>();
outFrame = ctx.allocateFrame();
@@ -114,13 +111,32 @@
for (int i = 0; i < framesLimit - 1; ++i) {
inFrames.add(ctx.allocateFrame());
}
- while (runs.size() > 0) {
- try {
- doPass(runs);
- } catch (Exception e) {
- throw new HyracksDataException(e);
+ int maxMergeWidth = framesLimit - 1;
+ while (runs.size() > maxMergeWidth) {
+ int generationSeparator = 0;
+ while (generationSeparator < runs.size() && runs.size() > maxMergeWidth) {
+ int mergeWidth = Math.min(Math.min(runs.size() - generationSeparator, maxMergeWidth),
+ runs.size() - maxMergeWidth + 1);
+ FileReference newRun = ctx.createManagedWorkspaceFile(ExternalSortRunMerger.class
+ .getSimpleName());
+ IFrameWriter mergeResultWriter = new RunFileWriter(newRun, ctx.getIOManager());
+ mergeResultWriter.open();
+ IFrameReader[] runCursors = new RunFileReader[mergeWidth];
+ for (int i = 0; i < mergeWidth; i++) {
+ runCursors[i] = runs.get(generationSeparator + i);
+ }
+ merge(mergeResultWriter, runCursors);
+ runs.subList(generationSeparator, mergeWidth + generationSeparator).clear();
+ runs.add(generationSeparator++, ((RunFileWriter) mergeResultWriter).createReader());
}
}
+ if (!runs.isEmpty()) {
+ IFrameReader[] runCursors = new RunFileReader[runs.size()];
+ for (int i = 0; i < runCursors.length; i++) {
+ runCursors[i] = runs.get(i);
+ }
+ merge(writer, runCursors);
+ }
}
} catch (Exception e) {
writer.fail();
@@ -130,44 +146,16 @@
}
}
- // creates a new run from runs that can fit in memory.
- private void doPass(List<IFrameReader> runs) throws HyracksDataException {
- FileReference newRun = null;
- IFrameWriter writer = this.writer;
- boolean finalPass = false;
- if (runs.size() + 1 <= framesLimit) { // + 1 outFrame
- finalPass = true;
- for (int i = inFrames.size() - 1; i >= runs.size(); i--) {
- inFrames.remove(i);
- }
- } else {
- newRun = ctx.createManagedWorkspaceFile(ExternalSortRunMerger.class.getSimpleName());
- writer = new RunFileWriter(newRun, ctx.getIOManager());
- writer.open();
- }
+ private void merge(IFrameWriter mergeResultWriter, IFrameReader[] runCursors) throws HyracksDataException {
+ RunMergingFrameReader merger = new RunMergingFrameReader(ctx, runCursors, inFrames, sortFields, comparators,
+ recordDesc);
+ merger.open();
try {
- IFrameReader[] runCursors = new RunFileReader[inFrames.size()];
- for (int i = 0; i < inFrames.size(); i++) {
- runCursors[i] = runs.get(i);
- }
- RunMergingFrameReader merger = new RunMergingFrameReader(ctx, runCursors, inFrames, sortFields,
- comparators, recordDesc);
- merger.open();
- try {
- while (merger.nextFrame(outFrame)) {
- FrameUtils.flushFrame(outFrame, writer);
- }
- } finally {
- merger.close();
- }
- runs.subList(0, inFrames.size()).clear();
- if (!finalPass) {
- runs.add(0, ((RunFileWriter) writer).createReader());
+ while (merger.nextFrame(outFrame)) {
+ FrameUtils.flushFrame(outFrame, mergeResultWriter);
}
} finally {
- if (!finalPass) {
- writer.close();
- }
+ merger.close();
}
}
@@ -186,7 +174,6 @@
FrameUtils.flushFrame(nextFrame, writer);
outFrameAppender.reset(nextFrame, true);
}
- System.gc();
return;
}
// Limit on the output size
@@ -216,11 +203,9 @@
FrameUtils.flushFrame(outFrame, writer);
outFrameAppender.reset(outFrame, true);
}
- System.gc();
return;
}
// More than one run, actual merging is needed
- System.gc();
inFrames = new ArrayList<ByteBuffer>();
for (int i = 0; i < framesLimit - 1; ++i) {
inFrames.add(ctx.allocateFrame());
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/FrameSorter.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/FrameSorter.java
index 3de9742..680b98e 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/FrameSorter.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/FrameSorter.java
@@ -19,7 +19,7 @@
import java.util.List;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.context.IHyracksCommonContext;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputer;
@@ -31,7 +31,7 @@
import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
public class FrameSorter {
- private final IHyracksCommonContext ctx;
+ private final IHyracksTaskContext ctx;
private final int[] sortFields;
private final INormalizedKeyComputer nkc;
private final IBinaryComparator[] comparators;
@@ -48,7 +48,7 @@
private int[] tPointers;
private int tupleCount;
- public FrameSorter(IHyracksCommonContext ctx, int[] sortFields,
+ public FrameSorter(IHyracksTaskContext ctx, int[] sortFields,
INormalizedKeyComputerFactory firstKeyNormalizerFactory, IBinaryComparatorFactory[] comparatorFactories,
RecordDescriptor recordDescriptor) {
this.ctx = ctx;
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/InMemorySortOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/InMemorySortOperatorDescriptor.java
index 30bebe9..3b3c28d 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/InMemorySortOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/InMemorySortOperatorDescriptor.java
@@ -29,11 +29,11 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractTaskState;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractStateObject;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
@@ -47,12 +47,12 @@
private INormalizedKeyComputerFactory firstKeyNormalizerFactory;
private IBinaryComparatorFactory[] comparatorFactories;
- public InMemorySortOperatorDescriptor(JobSpecification spec, int[] sortFields,
+ public InMemorySortOperatorDescriptor(IOperatorDescriptorRegistry spec, int[] sortFields,
IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDescriptor) {
this(spec, sortFields, null, comparatorFactories, recordDescriptor);
}
- public InMemorySortOperatorDescriptor(JobSpecification spec, int[] sortFields,
+ public InMemorySortOperatorDescriptor(IOperatorDescriptorRegistry spec, int[] sortFields,
INormalizedKeyComputerFactory firstKeyNormalizerFactory, IBinaryComparatorFactory[] comparatorFactories,
RecordDescriptor recordDescriptor) {
super(spec, 1, 1);
@@ -67,16 +67,16 @@
SortActivity sa = new SortActivity(new ActivityId(odId, SORT_ACTIVITY_ID));
MergeActivity ma = new MergeActivity(new ActivityId(odId, MERGE_ACTIVITY_ID));
- builder.addActivity(sa);
+ builder.addActivity(this, sa);
builder.addSourceEdge(0, sa, 0);
- builder.addActivity(ma);
+ builder.addActivity(this, ma);
builder.addTargetEdge(0, ma, 0);
builder.addBlockingEdge(sa, ma);
}
- public static class SortTaskState extends AbstractTaskState {
+ public static class SortTaskState extends AbstractStateObject {
private FrameSorter frameSorter;
public SortTaskState() {
@@ -124,7 +124,7 @@
@Override
public void close() throws HyracksDataException {
state.frameSorter.sortFrames();
- ctx.setTaskState(state);
+ ctx.setStateObject(state);
}
@Override
@@ -150,7 +150,7 @@
public void initialize() throws HyracksDataException {
writer.open();
try {
- SortTaskState state = (SortTaskState) ctx.getTaskState(new TaskId(new ActivityId(
+ SortTaskState state = (SortTaskState) ctx.getStateObject(new TaskId(new ActivityId(
getOperatorId(), SORT_ACTIVITY_ID), partition));
state.frameSorter.flushFrames(writer);
} catch (Exception e) {
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/OptimizedExternalSortOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/OptimizedExternalSortOperatorDescriptor.java
index a8653d1..d9c16d8 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/OptimizedExternalSortOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/OptimizedExternalSortOperatorDescriptor.java
@@ -32,11 +32,11 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractTaskState;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractStateObject;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
@@ -68,17 +68,17 @@
private final int memSize;
private final int outputLimit;
- public OptimizedExternalSortOperatorDescriptor(JobSpecification spec, int framesLimit, int[] sortFields,
+ public OptimizedExternalSortOperatorDescriptor(IOperatorDescriptorRegistry spec, int framesLimit, int[] sortFields,
IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDescriptor) {
this(spec, framesLimit, NO_LIMIT, sortFields, null, comparatorFactories, recordDescriptor);
}
- public OptimizedExternalSortOperatorDescriptor(JobSpecification spec, int framesLimit, int outputLimit,
+ public OptimizedExternalSortOperatorDescriptor(IOperatorDescriptorRegistry spec, int framesLimit, int outputLimit,
int[] sortFields, IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDescriptor) {
this(spec, framesLimit, outputLimit, sortFields, null, comparatorFactories, recordDescriptor);
}
- public OptimizedExternalSortOperatorDescriptor(JobSpecification spec, int memSize, int outputLimit,
+ public OptimizedExternalSortOperatorDescriptor(IOperatorDescriptorRegistry spec, int memSize, int outputLimit,
int[] sortFields, INormalizedKeyComputerFactory firstKeyNormalizerFactory,
IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDescriptor) {
super(spec, 1, 1);
@@ -98,16 +98,16 @@
OptimizedSortActivity osa = new OptimizedSortActivity(new ActivityId(odId, SORT_ACTIVITY_ID));
OptimizedMergeActivity oma = new OptimizedMergeActivity(new ActivityId(odId, MERGE_ACTIVITY_ID));
- builder.addActivity(osa);
+ builder.addActivity(this, osa);
builder.addSourceEdge(0, osa, 0);
- builder.addActivity(oma);
+ builder.addActivity(this, oma);
builder.addTargetEdge(0, oma, 0);
builder.addBlockingEdge(osa, oma);
}
- public static class OptimizedSortTaskState extends AbstractTaskState {
+ public static class OptimizedSortTaskState extends AbstractStateObject {
private List<IFrameReader> runs;
public OptimizedSortTaskState() {
@@ -165,7 +165,7 @@
new TaskId(getActivityId(), partition));
runGen.close();
state.runs = runGen.getRuns();
- ctx.setTaskState(state);
+ ctx.setStateObject(state);
}
@@ -191,8 +191,8 @@
IOperatorNodePushable op = new AbstractUnaryOutputSourceOperatorNodePushable() {
@Override
public void initialize() throws HyracksDataException {
- OptimizedSortTaskState state = (OptimizedSortTaskState) ctx.getTaskState(new TaskId(new ActivityId(
- getOperatorId(), SORT_ACTIVITY_ID), partition));
+ OptimizedSortTaskState state = (OptimizedSortTaskState) ctx.getStateObject(new TaskId(
+ new ActivityId(getOperatorId(), SORT_ACTIVITY_ID), partition));
List<IFrameReader> runs = state.runs;
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/RunMergingFrameReader.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/RunMergingFrameReader.java
index c014ed1..29c28a1 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/RunMergingFrameReader.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/RunMergingFrameReader.java
@@ -54,11 +54,11 @@
@Override
public void open() throws HyracksDataException {
- tupleAccessors = new FrameTupleAccessor[inFrames.size()];
+ tupleAccessors = new FrameTupleAccessor[runCursors.length];
Comparator<ReferenceEntry> comparator = createEntryComparator(comparators);
- topTuples = new ReferencedPriorityQueue(ctx.getFrameSize(), recordDesc, inFrames.size(), comparator);
- tupleIndexes = new int[inFrames.size()];
- for (int i = 0; i < inFrames.size(); i++) {
+ topTuples = new ReferencedPriorityQueue(ctx.getFrameSize(), recordDesc, runCursors.length, comparator);
+ tupleIndexes = new int[runCursors.length];
+ for (int i = 0; i < runCursors.length; i++) {
tupleIndexes[i] = 0;
int runIndex = topTuples.peek().getRunid();
runCursors[runIndex].open();
@@ -98,7 +98,7 @@
@Override
public void close() throws HyracksDataException {
- for (int i = 0; i < inFrames.size(); ++i) {
+ for (int i = 0; i < runCursors.length; ++i) {
closeRun(i, runCursors, tupleAccessors);
}
}
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/union/UnionAllOperatorDescriptor.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/union/UnionAllOperatorDescriptor.java
index f55eb3d..7ac9f58 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/union/UnionAllOperatorDescriptor.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/union/UnionAllOperatorDescriptor.java
@@ -24,13 +24,13 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputOperatorNodePushable;
public class UnionAllOperatorDescriptor extends AbstractOperatorDescriptor {
- public UnionAllOperatorDescriptor(JobSpecification spec, int nInputs, RecordDescriptor recordDescriptor) {
+ public UnionAllOperatorDescriptor(IOperatorDescriptorRegistry spec, int nInputs, RecordDescriptor recordDescriptor) {
super(spec, nInputs, 1);
recordDescriptors[0] = recordDescriptor;
}
@@ -40,7 +40,7 @@
@Override
public void contributeActivities(IActivityGraphBuilder builder) {
UnionActivityNode uba = new UnionActivityNode(new ActivityId(getOperatorId(), 0));
- builder.addActivity(uba);
+ builder.addActivity(this, uba);
for (int i = 0; i < inputArity; ++i) {
builder.addSourceEdge(i, uba, i);
}
@@ -58,7 +58,7 @@
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions)
throws HyracksDataException {
- RecordDescriptor inRecordDesc = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0);
+ RecordDescriptor inRecordDesc = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);
return new UnionOperator(ctx, inRecordDesc);
}
}
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
index 5a6d8cd..b6e8c72 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
@@ -44,15 +44,12 @@
import edu.uci.ics.hyracks.examples.btree.helper.IndexRegistryProvider;
import edu.uci.ics.hyracks.examples.btree.helper.StorageManagerInterface;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexInsertUpdateDeleteOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
// This example will insert tuples into the primary and secondary index using an insert pipeline
@@ -62,7 +59,7 @@
@Option(name = "-host", usage = "Hyracks Cluster Controller Host name", required = true)
public String host;
- @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
+ @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1098)")
public int port = 1098;
@Option(name = "-app", usage = "Hyracks Application name", required = true)
@@ -91,8 +88,7 @@
JobSpecification job = createJob(options);
long start = System.currentTimeMillis();
- JobId jobId = hcc.createJob(options.app, job);
- hcc.start(jobId);
+ JobId jobId = hcc.startJob(options.app, job);
hcc.waitForCompletion(jobId);
long end = System.currentTimeMillis();
System.err.println(start + " " + end + " " + (end - start));
@@ -144,11 +140,6 @@
IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[1];
primaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- // create factories and providers for secondary B-Tree
- TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
- ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(primaryTupleWriterFactory);
- ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
-
// the B-Tree expects its keyfields to be at the front of its input
// tuple
int[] primaryFieldPermutation = { 2, 1, 3, 4 }; // map field 2 of input
@@ -160,9 +151,9 @@
// create operator descriptor
TreeIndexInsertUpdateDeleteOperatorDescriptor primaryInsert = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
- spec, recDesc, storageManager, indexRegistryProvider, primarySplitProvider,
- primaryInteriorFrameFactory, primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories,
- primaryFieldPermutation, IndexOp.INSERT, dataflowHelperFactory);
+ spec, recDesc, storageManager, indexRegistryProvider, primarySplitProvider, primaryTypeTraits,
+ primaryComparatorFactories, primaryFieldPermutation, IndexOp.INSERT, dataflowHelperFactory, null,
+ NoOpOperationCallbackProvider.INSTANCE);
JobHelper.createPartitionConstraint(spec, primaryInsert, splitNCs);
// prepare insertion into secondary index
@@ -177,12 +168,6 @@
secondaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
secondaryComparatorFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- // create factories and providers for secondary B-Tree
- TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(secondaryTypeTraits);
- ITreeIndexFrameFactory secondaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- secondaryTupleWriterFactory);
- ITreeIndexFrameFactory secondaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(secondaryTupleWriterFactory);
-
// the B-Tree expects its keyfields to be at the front of its input
// tuple
int[] secondaryFieldPermutation = { 1, 2 };
@@ -190,9 +175,9 @@
options.secondaryBTreeName);
// create operator descriptor
TreeIndexInsertUpdateDeleteOperatorDescriptor secondaryInsert = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
- spec, recDesc, storageManager, indexRegistryProvider, secondarySplitProvider,
- secondaryInteriorFrameFactory, secondaryLeafFrameFactory, secondaryTypeTraits,
- secondaryComparatorFactories, secondaryFieldPermutation, IndexOp.INSERT, dataflowHelperFactory);
+ spec, recDesc, storageManager, indexRegistryProvider, secondarySplitProvider, secondaryTypeTraits,
+ secondaryComparatorFactories, secondaryFieldPermutation, IndexOp.INSERT, dataflowHelperFactory, null,
+ NoOpOperationCallbackProvider.INSTANCE);
JobHelper.createPartitionConstraint(spec, secondaryInsert, splitNCs);
// end the insert pipeline at this sink operator
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
index 0b1cd6c..a6c7ea6 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
@@ -43,14 +43,11 @@
import edu.uci.ics.hyracks.examples.btree.helper.IndexRegistryProvider;
import edu.uci.ics.hyracks.examples.btree.helper.StorageManagerInterface;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
// This example will load a primary index from randomly generated data
@@ -60,7 +57,7 @@
@Option(name = "-host", usage = "Hyracks Cluster Controller Host name", required = true)
public String host;
- @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
+ @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1098)")
public int port = 1098;
@Option(name = "-app", usage = "Hyracks Application name", required = true)
@@ -89,8 +86,7 @@
JobSpecification job = createJob(options);
long start = System.currentTimeMillis();
- JobId jobId = hcc.createJob(options.app, job);
- hcc.start(jobId);
+ JobId jobId = hcc.startJob(options.app, job);
hcc.waitForCompletion(jobId);
long end = System.currentTimeMillis();
System.err.println(start + " " + end + " " + (end - start));
@@ -144,10 +140,7 @@
typeTraits[2] = IntegerPointable.TYPE_TRAITS;
typeTraits[3] = UTF8StringPointable.TYPE_TRAITS;
- // create factories and providers for B-Tree
- TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ // create providers for B-Tree
IIndexRegistryProvider<IIndex> indexRegistryProvider = IndexRegistryProvider.INSTANCE;
IStorageManagerInterface storageManager = StorageManagerInterface.INSTANCE;
@@ -159,8 +152,8 @@
IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName);
IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, indexRegistryProvider, btreeSplitProvider, interiorFrameFactory, leafFrameFactory,
- typeTraits, comparatorFactories, fieldPermutation, 0.7f, dataflowHelperFactory);
+ storageManager, indexRegistryProvider, btreeSplitProvider, typeTraits, comparatorFactories,
+ fieldPermutation, 0.7f, dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
JobHelper.createPartitionConstraint(spec, btreeBulkLoad, splitNCs);
// distribute the records from the datagen via hashing to the bulk load
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
index aa73a62..d24ba33 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
@@ -42,13 +42,10 @@
import edu.uci.ics.hyracks.examples.btree.helper.StorageManagerInterface;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
// This example will perform an ordered scan on the primary index
@@ -59,7 +56,7 @@
@Option(name = "-host", usage = "Hyracks Cluster Controller Host name", required = true)
public String host;
- @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
+ @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1098)")
public int port = 1098;
@Option(name = "-app", usage = "Hyracks Application name", required = true)
@@ -82,8 +79,7 @@
JobSpecification job = createJob(options);
long start = System.currentTimeMillis();
- JobId jobId = hcc.createJob(options.app, job);
- hcc.start(jobId);
+ JobId jobId = hcc.startJob(options.app, job);
hcc.waitForCompletion(jobId);
long end = System.currentTimeMillis();
System.err.println(start + " " + end + " " + (end - start));
@@ -106,10 +102,7 @@
IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[1];
comparatorFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- // create factories and providers for B-Tree
- TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ // create roviders for B-Tree
IIndexRegistryProvider<IIndex> indexRegistryProvider = IndexRegistryProvider.INSTANCE;
IStorageManagerInterface storageManager = StorageManagerInterface.INSTANCE;
@@ -149,8 +142,8 @@
IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName);
IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
BTreeSearchOperatorDescriptor btreeSearchOp = new BTreeSearchOperatorDescriptor(spec, recDesc, storageManager,
- indexRegistryProvider, btreeSplitProvider, interiorFrameFactory, leafFrameFactory, typeTraits,
- comparatorFactories, true, lowKeyFields, highKeyFields, true, true, dataflowHelperFactory);
+ indexRegistryProvider, btreeSplitProvider, typeTraits, comparatorFactories, lowKeyFields,
+ highKeyFields, true, true, dataflowHelperFactory, false, NoOpOperationCallbackProvider.INSTANCE);
JobHelper.createPartitionConstraint(spec, btreeSearchOp, splitNCs);
// have each node print the results of its respective B-Tree
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
index ad9ad3a..5aa338a 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
@@ -36,15 +36,12 @@
import edu.uci.ics.hyracks.examples.btree.helper.IndexRegistryProvider;
import edu.uci.ics.hyracks.examples.btree.helper.StorageManagerInterface;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDiskOrderScanOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
// This example will load a secondary index with <key, primary-index key> pairs
@@ -55,7 +52,7 @@
@Option(name = "-host", usage = "Hyracks Cluster Controller Host name", required = true)
public String host;
- @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
+ @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1098)")
public int port = 1098;
@Option(name = "-app", usage = "Hyracks Application name", required = true)
@@ -84,8 +81,7 @@
JobSpecification job = createJob(options);
long start = System.currentTimeMillis();
- JobId jobId = hcc.createJob(options.app, job);
- hcc.start(jobId);
+ JobId jobId = hcc.startJob(options.app, job);
hcc.waitForCompletion(jobId);
long end = System.currentTimeMillis();
System.err.println(start + " " + end + " " + (end - start));
@@ -122,17 +118,12 @@
comparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
comparatorFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- // create factories and providers for primary B-Tree
- TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
- ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(primaryTupleWriterFactory);
- ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
-
// use a disk-order scan to read primary index
IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName);
IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
TreeIndexDiskOrderScanOperatorDescriptor btreeScanOp = new TreeIndexDiskOrderScanOperatorDescriptor(spec,
- recDesc, storageManager, indexRegistryProvider, primarySplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, dataflowHelperFactory);
+ recDesc, storageManager, indexRegistryProvider, primarySplitProvider, primaryTypeTraits,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
JobHelper.createPartitionConstraint(spec, btreeScanOp, splitNCs);
// sort the tuples as preparation for bulk load into secondary index
@@ -148,20 +139,13 @@
secondaryTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
secondaryTypeTraits[1] = IntegerPointable.TYPE_TRAITS;
- // create factories and providers for secondary B-Tree
- TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(secondaryTypeTraits);
- ITreeIndexFrameFactory secondaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- secondaryTupleWriterFactory);
- ITreeIndexFrameFactory secondaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(secondaryTupleWriterFactory);
-
// the B-Tree expects its keyfields to be at the front of its input
// tuple
int[] fieldPermutation = { 1, 0 };
IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.secondaryBTreeName);
TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, indexRegistryProvider, btreeSplitProvider, secondaryInteriorFrameFactory,
- secondaryLeafFrameFactory, secondaryTypeTraits, comparatorFactories, fieldPermutation, 0.7f,
- dataflowHelperFactory);
+ storageManager, indexRegistryProvider, btreeSplitProvider, secondaryTypeTraits, comparatorFactories,
+ fieldPermutation, 0.7f, dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
JobHelper.createPartitionConstraint(spec, btreeBulkLoad, splitNCs);
// connect the ops
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
index 2719397..277668b 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
@@ -42,13 +42,10 @@
import edu.uci.ics.hyracks.examples.btree.helper.StorageManagerInterface;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
// This example will perform range search on the secondary index
@@ -59,7 +56,7 @@
@Option(name = "-host", usage = "Hyracks Cluster Controller Host name", required = true)
public String host;
- @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
+ @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1098)")
public int port = 1098;
@Option(name = "-app", usage = "Hyracks Application name", required = true)
@@ -85,8 +82,7 @@
JobSpecification job = createJob(options);
long start = System.currentTimeMillis();
- JobId jobId = hcc.createJob(options.app, job);
- hcc.start(jobId);
+ JobId jobId = hcc.startJob(options.app, job);
hcc.waitForCompletion(jobId);
long end = System.currentTimeMillis();
System.err.println(start + " " + end + " " + (end - start));
@@ -119,12 +115,6 @@
IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[1];
primaryComparatorFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- // create factories and providers for secondary B-Tree
- TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(secondaryTypeTraits);
- ITreeIndexFrameFactory secondaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- secondaryTupleWriterFactory);
- ITreeIndexFrameFactory secondaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(secondaryTupleWriterFactory);
-
// schema of tuples coming out of primary index
RecordDescriptor primaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
@@ -137,11 +127,6 @@
primaryTypeTraits[2] = IntegerPointable.TYPE_TRAITS;
primaryTypeTraits[3] = UTF8StringPointable.TYPE_TRAITS;
- // create factories and providers for secondary B-Tree
- TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
- ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(primaryTupleWriterFactory);
- ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
-
// comparators for btree, note that we only need a comparator for the
// non-unique key
// i.e. we will have a range condition on the first field only (implying
@@ -183,9 +168,9 @@
options.secondaryBTreeName);
IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
BTreeSearchOperatorDescriptor secondarySearchOp = new BTreeSearchOperatorDescriptor(spec, secondaryRecDesc,
- storageManager, indexRegistryProvider, secondarySplitProvider, secondaryInteriorFrameFactory,
- secondaryLeafFrameFactory, secondaryTypeTraits, searchComparatorFactories, true, secondaryLowKeyFields,
- secondaryHighKeyFields, true, true, dataflowHelperFactory);
+ storageManager, indexRegistryProvider, secondarySplitProvider, secondaryTypeTraits,
+ searchComparatorFactories, secondaryLowKeyFields, secondaryHighKeyFields, true, true,
+ dataflowHelperFactory, false, NoOpOperationCallbackProvider.INSTANCE);
JobHelper.createPartitionConstraint(spec, secondarySearchOp, splitNCs);
// secondary index will output tuples with [UTF8String, Integer]
@@ -199,9 +184,9 @@
IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName);
BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
- storageManager, indexRegistryProvider, primarySplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, primaryLowKeyFields,
- primaryHighKeyFields, true, true, dataflowHelperFactory);
+ storageManager, indexRegistryProvider, primarySplitProvider, primaryTypeTraits,
+ primaryComparatorFactories, primaryLowKeyFields, primaryHighKeyFields, true, true,
+ dataflowHelperFactory, false, NoOpOperationCallbackProvider.INSTANCE);
JobHelper.createPartitionConstraint(spec, primarySearchOp, splitNCs);
// have each node print the results of its respective B-Tree
diff --git a/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/DataGenOperatorDescriptor.java b/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/DataGenOperatorDescriptor.java
index 36e315d..f342d23 100644
--- a/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/DataGenOperatorDescriptor.java
+++ b/hyracks/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/DataGenOperatorDescriptor.java
@@ -25,7 +25,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
@@ -45,7 +45,7 @@
private final int uniqueField;
private final long randomSeed;
- public DataGenOperatorDescriptor(JobSpecification spec, RecordDescriptor outputRecord, int numRecords,
+ public DataGenOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor outputRecord, int numRecords,
int uniqueField, int intMinVal, int intMaxVal, int maxStrLen, long randomSeed) {
super(spec, 0, 1);
this.numRecords = numRecords;
diff --git a/hyracks/hyracks-examples/hadoop-compat-example/hadoopcompatapp/pom.xml b/hyracks/hyracks-examples/hadoop-compat-example/hadoopcompatapp/pom.xml
index 7c90618..afe91d1 100644
--- a/hyracks/hyracks-examples/hadoop-compat-example/hadoopcompatapp/pom.xml
+++ b/hyracks/hyracks-examples/hadoop-compat-example/hadoopcompatapp/pom.xml
@@ -76,11 +76,12 @@
</plugin>
<plugin>
<groupId>edu.uci.ics.hyracks</groupId>
- <artifactId>hyracks-maven-plugin</artifactId>
- <version>0.0.2</version>
+ <artifactId>hyracks-virtualcluster-maven-plugin</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
<configuration>
<hyracksServerHome>${basedir}/../../../hyracks-server/target/hyracks-server-${project.version}-binary-assembly</hyracksServerHome>
<hyracksCLIHome>${basedir}/../../../hyracks-cli/target/hyracks-cli-${project.version}-binary-assembly</hyracksCLIHome>
+ <jvmOptions>${jvm.extraargs}</jvmOptions>
</configuration>
<executions>
<execution>
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexScanOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexScanOperatorTest.java
index 6ef1740..8482083 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexScanOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexScanOperatorTest.java
@@ -49,14 +49,12 @@
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
import edu.uci.ics.hyracks.test.support.TestIndexRegistryProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
@@ -80,10 +78,6 @@
private ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
private int primaryKeyFieldCount = 1;
private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
- private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- primaryTupleWriterFactory);
- private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
private static String primaryBtreeName = "primary" + simpleDateFormat.format(new Date());
private static String primaryFileName = System.getProperty("java.io.tmpdir") + sep + primaryBtreeName;
@@ -107,9 +101,20 @@
primaryTypeTraits[5] = UTF8StringPointable.TYPE_TRAITS;
primaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ createPrimaryIndex();
loadPrimaryIndexTest();
}
+ public void createPrimaryIndex() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ TreeIndexCreateOperatorDescriptor primaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager,
+ indexRegistryProvider, primaryBtreeSplitProvider, primaryTypeTraits, primaryComparatorFactories,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryCreateOp, NC1_ID);
+ spec.addRoot(primaryCreateOp);
+ runTest(spec);
+ }
+
public void loadPrimaryIndexTest() throws Exception {
JobSpecification spec = new JobSpecification();
@@ -138,9 +143,8 @@
int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
- dataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeBulkLoad, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
@@ -175,9 +179,8 @@
int[] highKeyFields = null; // + infinity
BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
- storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, lowKeyFields,
- highKeyFields, true, true, dataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryTypeTraits, primaryComparatorFactories, lowKeyFields,
+ highKeyFields, true, true, dataflowHelperFactory, false, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexSearchOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexSearchOperatorTest.java
index 51dbb69..82fecbe 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexSearchOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexSearchOperatorTest.java
@@ -49,14 +49,12 @@
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
import edu.uci.ics.hyracks.test.support.TestIndexRegistryProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
@@ -80,10 +78,6 @@
private ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
private int primaryKeyFieldCount = 1;
private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
- private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- primaryTupleWriterFactory);
- private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
private static String primaryBtreeName = "primary" + simpleDateFormat.format(new Date());
private static String primaryFileName = System.getProperty("java.io.tmpdir") + sep + primaryBtreeName;
@@ -107,9 +101,20 @@
primaryTypeTraits[5] = UTF8StringPointable.TYPE_TRAITS;
primaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ createPrimaryIndex();
loadPrimaryIndexTest();
}
+ public void createPrimaryIndex() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ TreeIndexCreateOperatorDescriptor primaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager,
+ indexRegistryProvider, primaryBtreeSplitProvider, primaryTypeTraits, primaryComparatorFactories,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryCreateOp, NC1_ID);
+ spec.addRoot(primaryCreateOp);
+ runTest(spec);
+ }
+
public void loadPrimaryIndexTest() throws Exception {
JobSpecification spec = new JobSpecification();
@@ -138,9 +143,8 @@
int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
- dataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeBulkLoad, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
@@ -180,9 +184,8 @@
int[] highKeyFields = { 1 };
BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
- storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, lowKeyFields,
- highKeyFields, true, true, dataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryTypeTraits, primaryComparatorFactories, lowKeyFields,
+ highKeyFields, true, true, dataflowHelperFactory, false, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexStatsOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexStatsOperatorTest.java
index a804bae..e63ce11 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexStatsOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexStatsOperatorTest.java
@@ -45,15 +45,13 @@
import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexStatsOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
import edu.uci.ics.hyracks.test.support.TestIndexRegistryProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
@@ -77,10 +75,6 @@
private ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
private int primaryKeyFieldCount = 1;
private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
- private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- primaryTupleWriterFactory);
- private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
private static String primaryBtreeName = "primary" + simpleDateFormat.format(new Date());
private static String primaryFileName = System.getProperty("java.io.tmpdir") + sep + primaryBtreeName;
@@ -88,10 +82,6 @@
private IFileSplitProvider primaryBtreeSplitProvider = new ConstantFileSplitProvider(
new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(primaryFileName))) });
- private RecordDescriptor primaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
@Before
public void setup() throws Exception {
@@ -104,9 +94,20 @@
primaryTypeTraits[5] = UTF8StringPointable.TYPE_TRAITS;
primaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ createPrimaryIndex();
loadPrimaryIndexTest();
}
+ public void createPrimaryIndex() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ TreeIndexCreateOperatorDescriptor primaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager,
+ indexRegistryProvider, primaryBtreeSplitProvider, primaryTypeTraits, primaryComparatorFactories,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryCreateOp, NC1_ID);
+ spec.addRoot(primaryCreateOp);
+ runTest(spec);
+ }
+
public void loadPrimaryIndexTest() throws Exception {
JobSpecification spec = new JobSpecification();
@@ -135,9 +136,8 @@
int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
- dataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeBulkLoad, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
@@ -153,8 +153,8 @@
JobSpecification spec = new JobSpecification();
TreeIndexStatsOperatorDescriptor primaryStatsOp = new TreeIndexStatsOperatorDescriptor(spec, storageManager,
- indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory, primaryLeafFrameFactory,
- primaryTypeTraits, primaryComparatorFactories, dataflowHelperFactory);
+ indexRegistryProvider, primaryBtreeSplitProvider,
+ primaryTypeTraits, primaryComparatorFactories, dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryStatsOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
createTempFile().getAbsolutePath()) });
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeSecondaryIndexInsertOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeSecondaryIndexInsertOperatorTest.java
index a16bbe4..3c87ae3 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeSecondaryIndexInsertOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeSecondaryIndexInsertOperatorTest.java
@@ -50,16 +50,14 @@
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexInsertUpdateDeleteOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
import edu.uci.ics.hyracks.test.support.TestIndexRegistryProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
@@ -83,10 +81,6 @@
private ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
private int primaryKeyFieldCount = 1;
private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
- private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- primaryTupleWriterFactory);
- private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
private static String primaryBtreeName = "primary" + simpleDateFormat.format(new Date());
private static String primaryFileName = System.getProperty("java.io.tmpdir") + sep + primaryBtreeName;
@@ -104,11 +98,6 @@
private ITypeTraits[] secondaryTypeTraits = new ITypeTraits[secondaryFieldCount];
private int secondaryKeyFieldCount = 2;
private IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[secondaryKeyFieldCount];
- private TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(
- secondaryTypeTraits);
- private ITreeIndexFrameFactory secondaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- secondaryTupleWriterFactory);
- private ITreeIndexFrameFactory secondaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(secondaryTupleWriterFactory);
private static String secondaryBtreeName = "secondary" + simpleDateFormat.format(new Date());
private static String secondaryFileName = System.getProperty("java.io.tmpdir") + sep + secondaryBtreeName;
@@ -136,11 +125,23 @@
secondaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
secondaryComparatorFactories[1] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ createPrimaryIndex();
loadPrimaryIndexTest();
+ createSecondaryIndex();
loadSecondaryIndexTest();
insertPipelineTest();
}
+ public void createPrimaryIndex() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ TreeIndexCreateOperatorDescriptor primaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager,
+ indexRegistryProvider, primaryBtreeSplitProvider, primaryTypeTraits, primaryComparatorFactories,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryCreateOp, NC1_ID);
+ spec.addRoot(primaryCreateOp);
+ runTest(spec);
+ }
+
public void loadPrimaryIndexTest() throws Exception {
JobSpecification spec = new JobSpecification();
@@ -169,9 +170,8 @@
int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
- dataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeBulkLoad, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
@@ -182,6 +182,16 @@
runTest(spec);
}
+ public void createSecondaryIndex() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ TreeIndexCreateOperatorDescriptor secondaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager,
+ indexRegistryProvider, secondaryBtreeSplitProvider, secondaryTypeTraits, secondaryComparatorFactories,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryCreateOp, NC1_ID);
+ spec.addRoot(secondaryCreateOp);
+ runTest(spec);
+ }
+
public void loadSecondaryIndexTest() throws Exception {
JobSpecification spec = new JobSpecification();
@@ -206,9 +216,8 @@
// scan primary index
BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
- storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, lowKeyFields,
- highKeyFields, true, true, dataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryTypeTraits, primaryComparatorFactories, lowKeyFields,
+ highKeyFields, true, true, dataflowHelperFactory, false, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
// sort based on secondary keys
@@ -220,9 +229,8 @@
// load secondary index
int[] fieldPermutation = { 3, 0 };
TreeIndexBulkLoadOperatorDescriptor secondaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, indexRegistryProvider, secondaryBtreeSplitProvider, secondaryInteriorFrameFactory,
- secondaryLeafFrameFactory, secondaryTypeTraits, secondaryComparatorFactories, fieldPermutation, 0.7f,
- dataflowHelperFactory);
+ storageManager, indexRegistryProvider, secondaryBtreeSplitProvider, secondaryTypeTraits, secondaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryBtreeBulkLoad, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryBtreeSearchOp, 0);
@@ -259,16 +267,16 @@
int[] primaryFieldPermutation = { 0, 1, 2, 4, 5, 7 };
TreeIndexInsertUpdateDeleteOperatorDescriptor primaryBtreeInsertOp = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
spec, ordersDesc, storageManager, indexRegistryProvider, primaryBtreeSplitProvider,
- primaryInteriorFrameFactory, primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories,
- primaryFieldPermutation, IndexOp.INSERT, dataflowHelperFactory);
+ primaryTypeTraits, primaryComparatorFactories,
+ primaryFieldPermutation, IndexOp.INSERT, dataflowHelperFactory, null, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeInsertOp, NC1_ID);
// first secondary index
int[] fieldPermutationB = { 4, 0 };
TreeIndexInsertUpdateDeleteOperatorDescriptor secondaryInsertOp = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
spec, ordersDesc, storageManager, indexRegistryProvider, secondaryBtreeSplitProvider,
- secondaryInteriorFrameFactory, secondaryLeafFrameFactory, secondaryTypeTraits,
- secondaryComparatorFactories, fieldPermutationB, IndexOp.INSERT, dataflowHelperFactory);
+ secondaryTypeTraits,
+ secondaryComparatorFactories, fieldPermutationB, IndexOp.INSERT, dataflowHelperFactory, null, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryInsertOp, NC1_ID);
NullSinkOperatorDescriptor nullSink = new NullSinkOperatorDescriptor(spec);
@@ -315,9 +323,9 @@
// search secondary index
BTreeSearchOperatorDescriptor secondaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec,
secondaryRecDesc, storageManager, indexRegistryProvider, secondaryBtreeSplitProvider,
- secondaryInteriorFrameFactory, secondaryLeafFrameFactory, secondaryTypeTraits,
- secondaryComparatorFactories, true, secondaryLowKeyFields, secondaryHighKeyFields, true, true,
- dataflowHelperFactory);
+ secondaryTypeTraits,
+ secondaryComparatorFactories, secondaryLowKeyFields, secondaryHighKeyFields, true, true,
+ dataflowHelperFactory, false, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryBtreeSearchOp, NC1_ID);
// second field from the tuples coming from secondary index
@@ -327,9 +335,8 @@
// search primary index
BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
- storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, primaryLowKeyFields,
- primaryHighKeyFields, true, true, dataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryTypeTraits, primaryComparatorFactories, primaryLowKeyFields,
+ primaryHighKeyFields, true, true, dataflowHelperFactory, false, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeSecondaryIndexSearchOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeSecondaryIndexSearchOperatorTest.java
index ef1c6f6..1304f12 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeSecondaryIndexSearchOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeSecondaryIndexSearchOperatorTest.java
@@ -49,14 +49,12 @@
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
import edu.uci.ics.hyracks.test.support.TestIndexRegistryProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
@@ -80,10 +78,6 @@
private ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
private int primaryKeyFieldCount = 1;
private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
- private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- primaryTupleWriterFactory);
- private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
private static String primaryBtreeName = "primary" + simpleDateFormat.format(new Date());
private static String primaryFileName = System.getProperty("java.io.tmpdir") + sep + primaryBtreeName;
@@ -101,11 +95,6 @@
private ITypeTraits[] secondaryTypeTraits = new ITypeTraits[secondaryFieldCount];
private int secondaryKeyFieldCount = 2;
private IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[secondaryKeyFieldCount];
- private TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(
- secondaryTypeTraits);
- private ITreeIndexFrameFactory secondaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- secondaryTupleWriterFactory);
- private ITreeIndexFrameFactory secondaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(secondaryTupleWriterFactory);
private static String secondaryBtreeName = "secondary" + simpleDateFormat.format(new Date());
private static String secondaryFileName = System.getProperty("java.io.tmpdir") + sep + secondaryBtreeName;
@@ -133,10 +122,22 @@
secondaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
secondaryComparatorFactories[1] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ createPrimaryIndex();
loadPrimaryIndexTest();
+ createSecondaryIndex();
loadSecondaryIndexTest();
}
+ public void createPrimaryIndex() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ TreeIndexCreateOperatorDescriptor primaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager,
+ indexRegistryProvider, primaryBtreeSplitProvider, primaryTypeTraits, primaryComparatorFactories,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryCreateOp, NC1_ID);
+ spec.addRoot(primaryCreateOp);
+ runTest(spec);
+ }
+
public void loadPrimaryIndexTest() throws Exception {
JobSpecification spec = new JobSpecification();
@@ -165,9 +166,8 @@
int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
- dataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeBulkLoad, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
@@ -178,6 +178,16 @@
runTest(spec);
}
+ public void createSecondaryIndex() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ TreeIndexCreateOperatorDescriptor secondaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager,
+ indexRegistryProvider, secondaryBtreeSplitProvider, secondaryTypeTraits, secondaryComparatorFactories,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryCreateOp, NC1_ID);
+ spec.addRoot(secondaryCreateOp);
+ runTest(spec);
+ }
+
public void loadSecondaryIndexTest() throws Exception {
JobSpecification spec = new JobSpecification();
@@ -202,9 +212,8 @@
// scan primary index
BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
- storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, lowKeyFields,
- highKeyFields, true, true, dataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryTypeTraits, primaryComparatorFactories, lowKeyFields,
+ highKeyFields, true, true, dataflowHelperFactory, false, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
// sort based on secondary keys
@@ -216,9 +225,8 @@
// load secondary index
int[] fieldPermutation = { 3, 0 };
TreeIndexBulkLoadOperatorDescriptor secondaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, indexRegistryProvider, secondaryBtreeSplitProvider, secondaryInteriorFrameFactory,
- secondaryLeafFrameFactory, secondaryTypeTraits, secondaryComparatorFactories, fieldPermutation, 0.7f,
- dataflowHelperFactory);
+ storageManager, indexRegistryProvider, secondaryBtreeSplitProvider, secondaryTypeTraits, secondaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryBtreeBulkLoad, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryBtreeSearchOp, 0);
@@ -260,9 +268,8 @@
// search secondary index
BTreeSearchOperatorDescriptor secondaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec,
secondaryRecDesc, storageManager, indexRegistryProvider, secondaryBtreeSplitProvider,
- secondaryInteriorFrameFactory, secondaryLeafFrameFactory, secondaryTypeTraits,
- secondaryComparatorFactories, true, secondaryLowKeyFields, secondaryHighKeyFields, true, true,
- dataflowHelperFactory);
+ secondaryTypeTraits, secondaryComparatorFactories, secondaryLowKeyFields, secondaryHighKeyFields, true, true,
+ dataflowHelperFactory, false, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryBtreeSearchOp, NC1_ID);
int[] primaryLowKeyFields = { 1 }; // second field from the tuples
@@ -272,9 +279,8 @@
// search primary index
BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
- storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, primaryLowKeyFields,
- primaryHighKeyFields, true, true, dataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryTypeTraits, primaryComparatorFactories, primaryLowKeyFields,
+ primaryHighKeyFields, true, true, dataflowHelperFactory, false, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
index 4ed8361..023bdd9 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
@@ -108,11 +108,10 @@
}
protected void runTest(JobSpecification spec) throws Exception {
- JobId jobId = hcc.createJob("test", spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info(spec.toJSON().toString(2));
}
- hcc.start(jobId);
+ JobId jobId = hcc.startJob("test", spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info(jobId.toString());
}
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
index dec42c1..e0b8c73 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
@@ -105,11 +105,10 @@
}
protected void runTest(JobSpecification spec) throws Exception {
- JobId jobId = hcc.createJob("test", spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info(spec.toJSON().toString(2));
}
- hcc.start(jobId);
+ JobId jobId = hcc.startJob("test", spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info(jobId.toString());
}
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AggregationTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AggregationTest.java
index 53a3f79..93e1e9b 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AggregationTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AggregationTest.java
@@ -26,6 +26,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
@@ -49,11 +50,8 @@
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.dataflow.std.file.ITupleParserFactory;
import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.group.ExternalGroupOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.group.HashGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.group.HashSpillableTableFactory;
import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.AvgFieldGroupAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.AvgFieldMergeAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.CountFieldAggregatorFactory;
@@ -61,6 +59,9 @@
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.IntSumFieldAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.MinMaxStringFieldAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.MultiFieldsAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.external.ExternalGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.hash.HashGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.preclustered.PreclusteredGroupOperatorDescriptor;
/**
*
@@ -107,7 +108,7 @@
UTF8StringParserFactory.INSTANCE, }, '|');
private AbstractSingleActivityOperatorDescriptor getPrinter(
- JobSpecification spec, String prefix) throws IOException {
+ IOperatorDescriptorRegistry spec, String prefix) throws IOException {
AbstractSingleActivityOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(
spec, new ConstantFileSplitProvider(new FileSplit[] {
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/CountOfCountsTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/CountOfCountsTest.java
index ce72ec5..aea6126 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/CountOfCountsTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/CountOfCountsTest.java
@@ -46,9 +46,9 @@
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.CountFieldAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.MultiFieldsAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.preclustered.PreclusteredGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.InMemorySortOperatorDescriptor;
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/LocalityAwareConnectorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/LocalityAwareConnectorTest.java
index a7a0a73..0d5a627 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/LocalityAwareConnectorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/LocalityAwareConnectorTest.java
@@ -27,6 +27,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
@@ -52,11 +53,11 @@
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.dataflow.std.file.ITupleParserFactory;
import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.group.HashGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.FloatSumFieldAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.IntSumFieldAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.MultiFieldsAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.hash.HashGroupOperatorDescriptor;
public class LocalityAwareConnectorTest extends AbstractMultiNCIntegrationTest {
@@ -86,8 +87,7 @@
/**
* Test of aggregations using locality aware connector. The output two files should be the
- * same, each of which is the aggregation of two copies of the lineitem.tbl.
- *
+ * same, each of which is the aggregation of two copies of the lineitem.tbl.
* Note that if the hashing connector is not working correctly, the two files may be different. This
* also means that even the output files are the same, the hashing may have other problems.
*
@@ -98,66 +98,47 @@
JobSpecification spec = new JobSpecification();
- FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
- spec, splitProvider, tupleParserFactory, desc);
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(spec, splitProvider, tupleParserFactory,
+ desc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- csvScanner, "asterix-001", "asterix-002", "asterix-003", "asterix-004");
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, csvScanner, "asterix-001", "asterix-002",
+ "asterix-003", "asterix-004");
- RecordDescriptor outputRec = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE,
- FloatSerializerDeserializer.INSTANCE });
+ RecordDescriptor outputRec = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, FloatSerializerDeserializer.INSTANCE });
int[] keyFields = new int[] { 0 };
int tableSize = 8;
- HashGroupOperatorDescriptor grouper = new HashGroupOperatorDescriptor(
- spec,
- keyFields,
- new FieldHashPartitionComputerFactory(
- keyFields,
+ HashGroupOperatorDescriptor grouper = new HashGroupOperatorDescriptor(spec, keyFields,
+ new FieldHashPartitionComputerFactory(keyFields,
new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
.of(UTF8StringPointable.FACTORY) }),
- new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
- .of(UTF8StringPointable.FACTORY) },
- new MultiFieldsAggregatorFactory(
- new IFieldAggregateDescriptorFactory[] {
- new IntSumFieldAggregatorFactory(1, true),
- new IntSumFieldAggregatorFactory(3, true),
- new FloatSumFieldAggregatorFactory(5, true) }),
- outputRec, tableSize);
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, true), new IntSumFieldAggregatorFactory(3, true),
+ new FloatSumFieldAggregatorFactory(5, true) }), outputRec, tableSize);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
- "asterix-005", "asterix-006");
-
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper, "asterix-005", "asterix-006");
+
BitSet nodemap = new BitSet(8);
-
+
nodemap.set(0);
nodemap.set(2);
nodemap.set(5);
nodemap.set(7);
- IConnectorDescriptor conn1 = new LocalityAwareMToNPartitioningConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(
- keyFields,
+ IConnectorDescriptor conn1 = new LocalityAwareMToNPartitioningConnectorDescriptor(spec,
+ new FieldHashPartitionComputerFactory(keyFields,
new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
.of(UTF8StringPointable.FACTORY) }), new HashtableLocalityMap(nodemap));
-
- new MToNPartitioningConnectorDescriptor(
- spec,
- new FieldHashPartitionComputerFactory(
- keyFields,
- new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
- .of(UTF8StringPointable.FACTORY) }));
+
spec.connect(conn1, csvScanner, 0, grouper, 0);
- AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
- "localityAwareSumInmemGroupTest");
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec, "localityAwareSumInmemGroupTest");
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
- "asterix-005", "asterix-006");
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, "asterix-005", "asterix-006");
IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
spec.connect(conn2, grouper, 0, printer, 0);
@@ -165,11 +146,11 @@
spec.addRoot(printer);
runTest(spec);
}
-
+
/**
* Test for locality aware connector, using the global hashing node mapper. This should have
* the exactly the same result as using {@link MToNPartitioningConnectorDescriptor}.
- *
+ *
* @throws Exception
*/
@Test
@@ -177,59 +158,40 @@
JobSpecification spec = new JobSpecification();
- FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
- spec, splitProvider, tupleParserFactory, desc);
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(spec, splitProvider, tupleParserFactory,
+ desc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- csvScanner, "asterix-001", "asterix-002", "asterix-003", "asterix-004");
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, csvScanner, "asterix-001", "asterix-002",
+ "asterix-003", "asterix-004");
- RecordDescriptor outputRec = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE,
- FloatSerializerDeserializer.INSTANCE });
+ RecordDescriptor outputRec = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, FloatSerializerDeserializer.INSTANCE });
int[] keyFields = new int[] { 0 };
int tableSize = 8;
- HashGroupOperatorDescriptor grouper = new HashGroupOperatorDescriptor(
- spec,
- keyFields,
- new FieldHashPartitionComputerFactory(
- keyFields,
+ HashGroupOperatorDescriptor grouper = new HashGroupOperatorDescriptor(spec, keyFields,
+ new FieldHashPartitionComputerFactory(keyFields,
new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
.of(UTF8StringPointable.FACTORY) }),
- new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
- .of(UTF8StringPointable.FACTORY) },
- new MultiFieldsAggregatorFactory(
- new IFieldAggregateDescriptorFactory[] {
- new IntSumFieldAggregatorFactory(1, true),
- new IntSumFieldAggregatorFactory(3, true),
- new FloatSumFieldAggregatorFactory(5, true) }),
- outputRec, tableSize);
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, true), new IntSumFieldAggregatorFactory(3, true),
+ new FloatSumFieldAggregatorFactory(5, true) }), outputRec, tableSize);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
- "asterix-005", "asterix-006");
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper, "asterix-005", "asterix-006");
- IConnectorDescriptor conn1 = new LocalityAwareMToNPartitioningConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(
- keyFields,
+ IConnectorDescriptor conn1 = new LocalityAwareMToNPartitioningConnectorDescriptor(spec,
+ new FieldHashPartitionComputerFactory(keyFields,
new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
.of(UTF8StringPointable.FACTORY) }), new GlobalHashingLocalityMap());
-
- new MToNPartitioningConnectorDescriptor(
- spec,
- new FieldHashPartitionComputerFactory(
- keyFields,
- new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
- .of(UTF8StringPointable.FACTORY) }));
+
spec.connect(conn1, csvScanner, 0, grouper, 0);
- AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
- "localityAwareSumInmemGroupTest");
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec, "localityAwareSumInmemGroupTest");
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
- "asterix-005", "asterix-006");
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, "asterix-005", "asterix-006");
IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
spec.connect(conn2, grouper, 0, printer, 0);
@@ -237,16 +199,14 @@
spec.addRoot(printer);
runTest(spec);
}
-
- private AbstractSingleActivityOperatorDescriptor getPrinter(
- JobSpecification spec, String prefix) throws IOException {
- AbstractSingleActivityOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(
- spec, new ConstantFileSplitProvider(new FileSplit[] {
- new FileSplit("asterix-005", createTempFile()
- .getAbsolutePath()),
- new FileSplit("asterix-006", createTempFile()
- .getAbsolutePath()) }), "\t");
+ private AbstractSingleActivityOperatorDescriptor getPrinter(IOperatorDescriptorRegistry spec, String prefix)
+ throws IOException {
+
+ AbstractSingleActivityOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec,
+ new ConstantFileSplitProvider(new FileSplit[] {
+ new FileSplit("asterix-005", createTempFile().getAbsolutePath()),
+ new FileSplit("asterix-006", createTempFile().getAbsolutePath()) }), "\t");
return printer;
}
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/OptimizedSortMergeTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/OptimizedSortMergeTest.java
index c2d3ebf..1ee4400 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/OptimizedSortMergeTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/OptimizedSortMergeTest.java
@@ -19,6 +19,7 @@
import org.junit.Test;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
@@ -39,8 +40,8 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.misc.LimitOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.OptimizedExternalSortOperatorDescriptor;
public class OptimizedSortMergeTest extends AbstractIntegrationTest {
@@ -74,7 +75,9 @@
PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) }, ordersDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID, NC2_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
@@ -125,7 +128,9 @@
LimitOperatorDescriptor filter = new LimitOperatorDescriptor(spec, ordersDesc, outputLimit);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, filter, NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java
index 6411390..1e60372 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java
@@ -20,6 +20,7 @@
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
@@ -60,7 +61,7 @@
}
@Override
- public ITuplePairComparator createTuplePairComparator() {
+ public ITuplePairComparator createTuplePairComparator(IHyracksTaskContext ctx) {
return new JoinComparator(bFactory.createBinaryComparator(), pos0, pos1);
}
}
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/BinaryTokenizerOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/BinaryTokenizerOperatorTest.java
index 76a60ae..836e72e 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/BinaryTokenizerOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/BinaryTokenizerOperatorTest.java
@@ -22,7 +22,6 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.invertedindex.dataflow.BinaryTokenizerOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.DelimitedUTF8StringBinaryTokenizerFactory;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizerFactory;
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/WordInvertedIndexTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/WordInvertedIndexTest.java
index 298c146..d8fd48e 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/WordInvertedIndexTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/WordInvertedIndexTest.java
@@ -20,6 +20,7 @@
import java.text.SimpleDateFormat;
import java.util.Date;
+import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
@@ -52,17 +53,16 @@
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifierFactory;
import edu.uci.ics.hyracks.storage.am.invertedindex.dataflow.BinaryTokenizerOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.invertedindex.dataflow.InvertedIndexBulkLoadOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.dataflow.InvertedIndexCreateOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.invertedindex.dataflow.InvertedIndexSearchOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers.ConjunctiveSearchModifierFactory;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.DelimitedUTF8StringBinaryTokenizerFactory;
@@ -86,10 +86,10 @@
private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
private final static String sep = System.getProperty("file.separator");
- private final String dateString = simpleDateFormat.format(new Date());
- private final String primaryFileName = System.getProperty("java.io.tmpdir") + sep + "primaryBtree" + dateString;
- private final String btreeFileName = System.getProperty("java.io.tmpdir") + sep + "invIndexBtree" + dateString;
- private final String invListsFileName = System.getProperty("java.io.tmpdir") + sep + "invIndexLists" + dateString;
+ private final static String dateString = simpleDateFormat.format(new Date());
+ private final static String primaryFileName = System.getProperty("java.io.tmpdir") + sep + "primaryBtree" + dateString;
+ private final static String btreeFileName = System.getProperty("java.io.tmpdir") + sep + "invIndexBtree" + dateString;
+ private final static String invListsFileName = System.getProperty("java.io.tmpdir") + sep + "invIndexLists" + dateString;
private IFileSplitProvider primaryFileSplitProvider = new ConstantFileSplitProvider(
new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(primaryFileName))) });
@@ -103,10 +103,6 @@
private ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
private int primaryKeyFieldCount = 1;
private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
- private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- primaryTupleWriterFactory);
- private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
private RecordDescriptor primaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
@@ -143,11 +139,23 @@
invListsTypeTraits[0] = IntegerPointable.TYPE_TRAITS;
invListsComparatorFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ createPrimaryIndex();
loadPrimaryIndex();
printPrimaryIndex();
+ createInvertedIndex();
loadInvertedIndex();
}
+ public void createPrimaryIndex() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ TreeIndexCreateOperatorDescriptor primaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager,
+ indexRegistryProvider, primaryFileSplitProvider, primaryTypeTraits, primaryComparatorFactories,
+ btreeDataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryCreateOp, NC1_ID);
+ spec.addRoot(primaryCreateOp);
+ runTest(spec);
+ }
+
@Test
public void testConjunctiveSearcher() throws Exception {
IInvertedIndexSearchModifierFactory conjunctiveSearchModifierFactory = new ConjunctiveSearchModifierFactory();
@@ -172,9 +180,8 @@
private IOperatorDescriptor createPrimaryBulkLoadOp(JobSpecification spec) {
int[] fieldPermutation = { 0, 1 };
TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, indexRegistryProvider, primaryFileSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
- btreeDataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryFileSplitProvider, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
+ btreeDataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeBulkLoad, NC1_ID);
return primaryBtreeBulkLoad;
}
@@ -199,9 +206,8 @@
int[] lowKeyFields = null; // - infinity
int[] highKeyFields = null; // + infinity
BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
- storageManager, indexRegistryProvider, primaryFileSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, lowKeyFields,
- highKeyFields, true, true, btreeDataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryFileSplitProvider, primaryTypeTraits, primaryComparatorFactories, lowKeyFields,
+ highKeyFields, true, true, btreeDataflowHelperFactory, false, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
return primaryBtreeSearchOp;
}
@@ -250,11 +256,24 @@
InvertedIndexBulkLoadOperatorDescriptor invIndexBulkLoadOp = new InvertedIndexBulkLoadOperatorDescriptor(spec,
fieldPermutation, storageManager, btreeFileSplitProvider, invListsFileSplitProvider,
indexRegistryProvider, tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits,
- invListsComparatorFactories, btreeDataflowHelperFactory);
+ invListsComparatorFactories, tokenizerFactory, btreeDataflowHelperFactory,
+ NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, invIndexBulkLoadOp, NC1_ID);
return invIndexBulkLoadOp;
}
+ public void createInvertedIndex() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ InvertedIndexCreateOperatorDescriptor invIndexCreateOp = new InvertedIndexCreateOperatorDescriptor(spec,
+ storageManager, btreeFileSplitProvider, invListsFileSplitProvider,
+ indexRegistryProvider, tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits,
+ invListsComparatorFactories, tokenizerFactory, btreeDataflowHelperFactory,
+ NoOpOperationCallbackProvider.INSTANCE);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, invIndexCreateOp, NC1_ID);
+ spec.addRoot(invIndexCreateOp);
+ runTest(spec);
+ }
+
public void loadInvertedIndex() throws Exception {
JobSpecification spec = new JobSpecification();
IOperatorDescriptor keyProviderOp = createScanKeyProviderOp(spec);
@@ -295,7 +314,8 @@
InvertedIndexSearchOperatorDescriptor invIndexSearchOp = new InvertedIndexSearchOperatorDescriptor(spec, 0,
storageManager, btreeFileSplitProvider, invListsFileSplitProvider, indexRegistryProvider,
tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits, invListsComparatorFactories,
- btreeDataflowHelperFactory, tokenizerFactory, searchModifierFactory, invListsRecDesc);
+ btreeDataflowHelperFactory, tokenizerFactory, searchModifierFactory, invListsRecDesc, false,
+ NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, invIndexSearchOp, NC1_ID);
return invIndexSearchOp;
}
@@ -313,4 +333,14 @@
spec.addRoot(printer);
runTest(spec);
}
+
+ @AfterClass
+ public static void cleanup() throws Exception {
+ File primary = new File(primaryFileName);
+ File btree = new File(btreeFileName);
+ File invLists = new File(invListsFileName);
+ primary.deleteOnExit();
+ btree.deleteOnExit();
+ invLists.deleteOnExit();
+ }
}
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreePrimaryIndexSearchOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreePrimaryIndexSearchOperatorTest.java
index 0c0df42..6625148 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreePrimaryIndexSearchOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreePrimaryIndexSearchOperatorTest.java
@@ -50,16 +50,14 @@
import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.rtree.dataflow.RTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.rtree.dataflow.RTreeSearchOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
import edu.uci.ics.hyracks.test.support.TestIndexRegistryProvider;
@@ -74,7 +72,7 @@
private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
private IIndexRegistryProvider<IIndex> indexRegistryProvider = new TestIndexRegistryProvider();
- private IIndexDataflowHelperFactory dataflowHelperFactory = new RTreeDataflowHelperFactory();
+ private IIndexDataflowHelperFactory dataflowHelperFactory;
private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
private final static String sep = System.getProperty("file.separator");
@@ -85,23 +83,19 @@
private ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private RTreeTypeAwareTupleWriterFactory primaryTupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(
- primaryTypeTraits);
-
private RecordDescriptor primaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
UTF8StringSerializerDeserializer.INSTANCE });
- private ITreeIndexFrameFactory primaryInteriorFrameFactory;
- private ITreeIndexFrameFactory primaryLeafFrameFactory;
-
private static String primaryRTreeName = "primary" + simpleDateFormat.format(new Date());
private static String primaryFileName = System.getProperty("java.io.tmpdir") + sep + primaryRTreeName;
private IFileSplitProvider primaryRTreeSplitProvider = new ConstantFileSplitProvider(
new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(primaryFileName))) });
+ private IPrimitiveValueProviderFactory[] primaryValueProviderFactories;
+
@Before
public void setup() throws Exception {
// field, type and key declarations for primary R-tree index
@@ -115,16 +109,24 @@
primaryComparatorFactories[2] = primaryComparatorFactories[0];
primaryComparatorFactories[3] = primaryComparatorFactories[0];
- IPrimitiveValueProviderFactory[] primaryValueProviderFactories = RTreeUtils
+ primaryValueProviderFactories = RTreeUtils
.createPrimitiveValueProviderFactories(primaryComparatorFactories.length, DoublePointable.FACTORY);
-
- primaryInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(primaryTupleWriterFactory,
- primaryValueProviderFactories);
- primaryLeafFrameFactory = new RTreeNSMLeafFrameFactory(primaryTupleWriterFactory, primaryValueProviderFactories);
-
+ dataflowHelperFactory = new RTreeDataflowHelperFactory(primaryValueProviderFactories);
+
+ createPrimaryIndex();
loadPrimaryIndexTest();
}
+ public void createPrimaryIndex() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ TreeIndexCreateOperatorDescriptor primaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager,
+ indexRegistryProvider, primaryRTreeSplitProvider, primaryTypeTraits, primaryComparatorFactories,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryCreateOp, NC1_ID);
+ spec.addRoot(primaryCreateOp);
+ runTest(spec);
+ }
+
public void loadPrimaryIndexTest() throws Exception {
JobSpecification spec = new JobSpecification();
@@ -144,9 +146,8 @@
int[] fieldPermutation = { 0, 1, 2, 3, 4 };
TreeIndexBulkLoadOperatorDescriptor primaryRTreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, indexRegistryProvider, primaryRTreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
- dataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryRTreeSplitProvider, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryRTreeBulkLoad, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), objScanner, 0, primaryRTreeBulkLoad, 0);
@@ -185,9 +186,8 @@
int[] keyFields = { 0, 1, 2, 3 };
RTreeSearchOperatorDescriptor primaryRTreeSearchOp = new RTreeSearchOperatorDescriptor(spec, primaryRecDesc,
- storageManager, indexRegistryProvider, primaryRTreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, keyFields,
- dataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryRTreeSplitProvider, primaryTypeTraits, primaryComparatorFactories, keyFields,
+ dataflowHelperFactory, false, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryRTreeSearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreePrimaryIndexStatsOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreePrimaryIndexStatsOperatorTest.java
index a4912e8..ef2950e 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreePrimaryIndexStatsOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreePrimaryIndexStatsOperatorTest.java
@@ -51,7 +51,9 @@
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexStatsOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.rtree.dataflow.RTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory;
@@ -70,7 +72,7 @@
private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
private IIndexRegistryProvider<IIndex> indexRegistryProvider = new TestIndexRegistryProvider();
- private IIndexDataflowHelperFactory dataflowHelperFactory = new RTreeDataflowHelperFactory();
+ private IIndexDataflowHelperFactory dataflowHelperFactory;
private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
private final static String sep = System.getProperty("file.separator");
@@ -98,6 +100,8 @@
private IFileSplitProvider primaryRTreeSplitProvider = new ConstantFileSplitProvider(
new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(primaryFileName))) });
+ private IPrimitiveValueProviderFactory[] primaryValueProviderFactories;
+
@Before
public void setup() throws Exception {
// field, type and key declarations for primary R-tree index
@@ -111,16 +115,28 @@
primaryComparatorFactories[2] = primaryComparatorFactories[0];
primaryComparatorFactories[3] = primaryComparatorFactories[0];
- IPrimitiveValueProviderFactory[] primaryValueProviderFactories = RTreeUtils
+ primaryValueProviderFactories = RTreeUtils
.createPrimitiveValueProviderFactories(primaryComparatorFactories.length, DoublePointable.FACTORY);
-
+ dataflowHelperFactory = new RTreeDataflowHelperFactory(primaryValueProviderFactories);
+
primaryInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(primaryTupleWriterFactory,
primaryValueProviderFactories);
primaryLeafFrameFactory = new RTreeNSMLeafFrameFactory(primaryTupleWriterFactory, primaryValueProviderFactories);
+ createPrimaryIndex();
loadPrimaryIndexTest();
}
+ public void createPrimaryIndex() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ TreeIndexCreateOperatorDescriptor primaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager,
+ indexRegistryProvider, primaryRTreeSplitProvider, primaryTypeTraits, primaryComparatorFactories,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryCreateOp, NC1_ID);
+ spec.addRoot(primaryCreateOp);
+ runTest(spec);
+ }
+
public void loadPrimaryIndexTest() throws Exception {
JobSpecification spec = new JobSpecification();
@@ -140,9 +156,8 @@
int[] fieldPermutation = { 0, 1, 2, 3, 4 };
TreeIndexBulkLoadOperatorDescriptor primaryRTreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, indexRegistryProvider, primaryRTreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
- dataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryRTreeSplitProvider, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryRTreeBulkLoad, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), objScanner, 0, primaryRTreeBulkLoad, 0);
@@ -156,8 +171,8 @@
JobSpecification spec = new JobSpecification();
TreeIndexStatsOperatorDescriptor primaryStatsOp = new TreeIndexStatsOperatorDescriptor(spec, storageManager,
- indexRegistryProvider, primaryRTreeSplitProvider, primaryInteriorFrameFactory, primaryLeafFrameFactory,
- primaryTypeTraits, primaryComparatorFactories, dataflowHelperFactory);
+ indexRegistryProvider, primaryRTreeSplitProvider,
+ primaryTypeTraits, primaryComparatorFactories, dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryStatsOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreeSecondaryIndexSearchOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreeSecondaryIndexSearchOperatorTest.java
index 025f675..030afcf 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreeSecondaryIndexSearchOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreeSecondaryIndexSearchOperatorTest.java
@@ -52,21 +52,15 @@
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.rtree.dataflow.RTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.rtree.dataflow.RTreeSearchOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.DoublePrimitiveValueProviderFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
import edu.uci.ics.hyracks.test.support.TestIndexRegistryProvider;
@@ -81,7 +75,7 @@
private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
private IIndexRegistryProvider<IIndex> indexRegistryProvider = new TestIndexRegistryProvider();
- private IIndexDataflowHelperFactory dataflowHelperFactory = new RTreeDataflowHelperFactory();
+ private IIndexDataflowHelperFactory dataflowHelperFactory;
private IIndexDataflowHelperFactory btreeDataflowHelperFactory = new BTreeDataflowHelperFactory();
private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
@@ -92,12 +86,6 @@
private ITypeTraits[] primaryBTreeTypeTraits = new ITypeTraits[primaryBTreeFieldCount];
private int primaryBTreeKeyFieldCount = 1;
private IBinaryComparatorFactory[] primaryBTreeComparatorFactories = new IBinaryComparatorFactory[primaryBTreeKeyFieldCount];
- private TypeAwareTupleWriterFactory primaryBTreeTupleWriterFactory = new TypeAwareTupleWriterFactory(
- primaryBTreeTypeTraits);
- private ITreeIndexFrameFactory primaryBTreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- primaryBTreeTupleWriterFactory);
- private ITreeIndexFrameFactory primaryBTreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(
- primaryBTreeTupleWriterFactory);
private static String primaryBTreeName = "primaryBTree" + simpleDateFormat.format(new Date());
private static String primaryBTreeFileName = System.getProperty("java.io.tmpdir") + sep + primaryBTreeName;
@@ -117,13 +105,6 @@
private ITypeTraits[] secondaryTypeTraits = new ITypeTraits[secondaryFieldCount];
private int secondaryKeyFieldCount = 4;
private IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[secondaryKeyFieldCount];
- private IPrimitiveValueProviderFactory[] secondaryValueProviderFactories = new IPrimitiveValueProviderFactory[secondaryKeyFieldCount];
-
- private RTreeTypeAwareTupleWriterFactory secondaryTupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(
- secondaryTypeTraits);
-
- private ITreeIndexFrameFactory secondaryInteriorFrameFactory;
- private ITreeIndexFrameFactory secondaryLeafFrameFactory;
private static String secondaryRTreeName = "secondary" + simpleDateFormat.format(new Date());
private static String secondaryFileName = System.getProperty("java.io.tmpdir") + sep + secondaryRTreeName;
@@ -136,6 +117,8 @@
DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
UTF8StringSerializerDeserializer.INSTANCE });
+ private IPrimitiveValueProviderFactory[] secondaryValueProviderFactories;
+
@Before
public void setup() throws Exception {
// field, type and key declarations for primary B-tree index
@@ -161,23 +144,28 @@
secondaryComparatorFactories[1] = secondaryComparatorFactories[0];
secondaryComparatorFactories[2] = secondaryComparatorFactories[0];
secondaryComparatorFactories[3] = secondaryComparatorFactories[0];
- secondaryValueProviderFactories[0] = DoublePrimitiveValueProviderFactory.INSTANCE;
- secondaryValueProviderFactories[1] = secondaryValueProviderFactories[0];
- secondaryValueProviderFactories[2] = secondaryValueProviderFactories[0];
- secondaryValueProviderFactories[3] = secondaryValueProviderFactories[0];
- IPrimitiveValueProviderFactory[] secondaryValueProviderFactories = RTreeUtils
+ secondaryValueProviderFactories = RTreeUtils
.createPrimitiveValueProviderFactories(secondaryComparatorFactories.length, DoublePointable.FACTORY);
- secondaryInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(secondaryTupleWriterFactory,
- secondaryValueProviderFactories);
- secondaryLeafFrameFactory = new RTreeNSMLeafFrameFactory(secondaryTupleWriterFactory,
- secondaryValueProviderFactories);
-
+ dataflowHelperFactory = new RTreeDataflowHelperFactory(secondaryValueProviderFactories);
+
+ createPrimaryIndex();
loadPrimaryBTreeIndexTest();
+ createSecondaryIndex();
loadSecondaryIndexTest();
}
+ public void createPrimaryIndex() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ TreeIndexCreateOperatorDescriptor primaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager,
+ indexRegistryProvider, primaryBTreeSplitProvider, primaryBTreeTypeTraits,
+ primaryBTreeComparatorFactories, btreeDataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryCreateOp, NC1_ID);
+ spec.addRoot(primaryCreateOp);
+ runTest(spec);
+ }
+
public void loadPrimaryBTreeIndexTest() throws Exception {
JobSpecification spec = new JobSpecification();
@@ -210,9 +198,8 @@
int[] fieldPermutation = { 0, 1, 2, 4, 5, 7, 9, 10, 11, 12 };
TreeIndexBulkLoadOperatorDescriptor primaryBTreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, indexRegistryProvider, primaryBTreeSplitProvider, primaryBTreeInteriorFrameFactory,
- primaryBTreeLeafFrameFactory, primaryBTreeTypeTraits, primaryBTreeComparatorFactories,
- fieldPermutation, 0.7f, btreeDataflowHelperFactory);
+ storageManager, indexRegistryProvider, primaryBTreeSplitProvider, primaryBTreeTypeTraits, primaryBTreeComparatorFactories,
+ fieldPermutation, 0.7f, btreeDataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBTreeBulkLoad, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
@@ -223,6 +210,16 @@
runTest(spec);
}
+ public void createSecondaryIndex() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ TreeIndexCreateOperatorDescriptor primaryCreateOp = new TreeIndexCreateOperatorDescriptor(spec, storageManager,
+ indexRegistryProvider, secondaryRTreeSplitProvider, secondaryTypeTraits, secondaryComparatorFactories,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryCreateOp, NC1_ID);
+ spec.addRoot(primaryCreateOp);
+ runTest(spec);
+ }
+
public void loadSecondaryIndexTest() throws Exception {
JobSpecification spec = new JobSpecification();
@@ -248,17 +245,15 @@
// scan primary index
BTreeSearchOperatorDescriptor primaryBTreeSearchOp = new BTreeSearchOperatorDescriptor(spec,
primaryBTreeRecDesc, storageManager, indexRegistryProvider, primaryBTreeSplitProvider,
- primaryBTreeInteriorFrameFactory, primaryBTreeLeafFrameFactory, primaryBTreeTypeTraits,
- primaryBTreeComparatorFactories, true, lowKeyFields, highKeyFields, true, true,
- btreeDataflowHelperFactory);
+ primaryBTreeTypeTraits, primaryBTreeComparatorFactories, lowKeyFields, highKeyFields,
+ true, true, btreeDataflowHelperFactory, false, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBTreeSearchOp, NC1_ID);
// load secondary index
int[] fieldPermutation = { 6, 7, 8, 9, 0 };
TreeIndexBulkLoadOperatorDescriptor secondaryRTreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, indexRegistryProvider, secondaryRTreeSplitProvider, secondaryInteriorFrameFactory,
- secondaryLeafFrameFactory, secondaryTypeTraits, secondaryComparatorFactories, fieldPermutation, 0.7f,
- dataflowHelperFactory);
+ storageManager, indexRegistryProvider, secondaryRTreeSplitProvider, secondaryTypeTraits, secondaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryRTreeBulkLoad, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryBTreeSearchOp, 0);
@@ -299,8 +294,7 @@
RTreeSearchOperatorDescriptor secondaryRTreeSearchOp = new RTreeSearchOperatorDescriptor(spec,
secondaryRecDesc, storageManager, indexRegistryProvider, secondaryRTreeSplitProvider,
- secondaryInteriorFrameFactory, secondaryLeafFrameFactory, secondaryTypeTraits,
- secondaryComparatorFactories, keyFields, dataflowHelperFactory);
+ secondaryTypeTraits, secondaryComparatorFactories, keyFields, dataflowHelperFactory, false, NoOpOperationCallbackProvider.INSTANCE);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryRTreeSearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
diff --git a/hyracks/hyracks-examples/text-example/textapp/pom.xml b/hyracks/hyracks-examples/text-example/textapp/pom.xml
index a9c4d0d..50b492b 100644
--- a/hyracks/hyracks-examples/text-example/textapp/pom.xml
+++ b/hyracks/hyracks-examples/text-example/textapp/pom.xml
@@ -76,11 +76,12 @@
</plugin>
<plugin>
<groupId>edu.uci.ics.hyracks</groupId>
- <artifactId>hyracks-maven-plugin</artifactId>
- <version>0.0.2</version>
+ <artifactId>hyracks-virtualcluster-maven-plugin</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
<configuration>
<hyracksServerHome>${basedir}/../../../hyracks-server/target/hyracks-server-${project.version}-binary-assembly</hyracksServerHome>
<hyracksCLIHome>${basedir}/../../../hyracks-cli/target/hyracks-cli-${project.version}-binary-assembly</hyracksCLIHome>
+ <jvmOptions>${jvm.extraargs}</jvmOptions>
</configuration>
<executions>
<execution>
diff --git a/hyracks/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/ExternalGroupClient.java b/hyracks/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/ExternalGroupClient.java
index 66f7efc..943f232 100644
--- a/hyracks/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/ExternalGroupClient.java
+++ b/hyracks/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/ExternalGroupClient.java
@@ -54,12 +54,12 @@
import edu.uci.ics.hyracks.dataflow.std.file.FrameFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.group.HashGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.group.HashSpillableTableFactory;
import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.CountFieldAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.IntSumFieldAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.MultiFieldsAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.hash.HashGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
/**
@@ -71,8 +71,8 @@
@Option(name = "-host", usage = "Hyracks Cluster Controller Host name", required = true)
public String host;
- @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
- public int port = 1099;
+ @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1098)")
+ public int port = 1098;
@Option(name = "-app", usage = "Hyracks Application name", required = true)
public String app;
@@ -122,8 +122,7 @@
System.out.print(i + "\t" + (System.currentTimeMillis() - start));
start = System.currentTimeMillis();
- JobId jobId = hcc.createJob(options.app, job);
- hcc.start(jobId);
+ JobId jobId = hcc.startJob(options.app, job);
hcc.waitForCompletion(jobId);
System.out.println("\t" + (System.currentTimeMillis() - start));
}
@@ -201,8 +200,8 @@
switch (alg) {
case 0: // new external hash graph
- grouper = new edu.uci.ics.hyracks.dataflow.std.group.ExternalGroupOperatorDescriptor(spec, keys,
- framesLimit, new IBinaryComparatorFactory[] {
+ grouper = new edu.uci.ics.hyracks.dataflow.std.group.external.ExternalGroupOperatorDescriptor(spec,
+ keys, framesLimit, new IBinaryComparatorFactory[] {
// PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY),
PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) },
new IntegerNormalizedKeyComputerFactory(), new MultiFieldsAggregatorFactory(
@@ -238,8 +237,8 @@
PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY) }));
spec.connect(scanSortConn2, fileScanner, 0, sorter2, 0);
- grouper = new edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupOperatorDescriptor(spec, keys,
- new IBinaryComparatorFactory[] {
+ grouper = new edu.uci.ics.hyracks.dataflow.std.group.preclustered.PreclusteredGroupOperatorDescriptor(
+ spec, keys, new IBinaryComparatorFactory[] {
// PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY),
PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) },
new MultiFieldsAggregatorFactory(
@@ -274,8 +273,8 @@
spec.connect(scanConn2, fileScanner, 0, grouper, 0);
break;
default:
- grouper = new edu.uci.ics.hyracks.dataflow.std.group.ExternalGroupOperatorDescriptor(spec, keys,
- framesLimit, new IBinaryComparatorFactory[] {
+ grouper = new edu.uci.ics.hyracks.dataflow.std.group.external.ExternalGroupOperatorDescriptor(spec,
+ keys, framesLimit, new IBinaryComparatorFactory[] {
// PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY),
PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) },
new IntegerNormalizedKeyComputerFactory(), new MultiFieldsAggregatorFactory(
diff --git a/hyracks/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java b/hyracks/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java
index 1708259..31019ab 100644
--- a/hyracks/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java
+++ b/hyracks/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java
@@ -48,11 +48,11 @@
import edu.uci.ics.hyracks.dataflow.std.file.FrameFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.group.HashGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.CountFieldAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.MultiFieldsAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.hash.HashGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.preclustered.PreclusteredGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.InMemorySortOperatorDescriptor;
import edu.uci.ics.hyracks.examples.text.WordTupleParserFactory;
@@ -62,7 +62,7 @@
@Option(name = "-host", usage = "Hyracks Cluster Controller Host name", required = true)
public String host;
- @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
+ @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1098)")
public int port = 1098;
@Option(name = "-app", usage = "Hyracks Application name", required = true)
@@ -101,9 +101,8 @@
options.algo, options.htSize, options.sbSize, options.format);
long start = System.currentTimeMillis();
- JobId jobId = hcc.createJob(options.app, job, options.runtimeProfiling ? EnumSet.of(JobFlag.PROFILE_RUNTIME)
+ JobId jobId = hcc.startJob(options.app, job, options.runtimeProfiling ? EnumSet.of(JobFlag.PROFILE_RUNTIME)
: EnumSet.noneOf(JobFlag.class));
- hcc.start(jobId);
hcc.waitForCompletion(jobId);
long end = System.currentTimeMillis();
System.err.println(start + " " + end + " " + (end - start));
diff --git a/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/edu/uci/ics/hyracks/examples/tpch/client/Main.java b/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/edu/uci/ics/hyracks/examples/tpch/client/Main.java
index 72533e7..01ccdef 100644
--- a/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/edu/uci/ics/hyracks/examples/tpch/client/Main.java
+++ b/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/edu/uci/ics/hyracks/examples/tpch/client/Main.java
@@ -24,6 +24,7 @@
import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
@@ -54,10 +55,10 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.FrameFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.dataflow.std.group.HashGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.CountFieldAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.group.aggregators.MultiFieldsAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.hash.HashGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.join.GraceHashJoinOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.join.HybridHashJoinOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.join.InMemoryHashJoinOperatorDescriptor;
@@ -68,7 +69,7 @@
@Option(name = "-host", usage = "Hyracks Cluster Controller Host name", required = true)
public String host;
- @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)", required = false)
+ @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1098)", required = false)
public int port = 1098;
@Option(name = "-app", usage = "Hyracks Application name", required = true)
@@ -126,9 +127,8 @@
options.graceFactor, options.memSize, options.tableSize, options.hasGroupBy);
long start = System.currentTimeMillis();
- JobId jobId = hcc.createJob(options.app, job,
+ JobId jobId = hcc.startJob(options.app, job,
options.profile ? EnumSet.of(JobFlag.PROFILE_RUNTIME) : EnumSet.noneOf(JobFlag.class));
- hcc.start(jobId);
hcc.waitForCompletion(jobId);
long end = System.currentTimeMillis();
System.err.println(start + " " + end + " " + (end - start));
@@ -315,7 +315,7 @@
}
@Override
- public ITuplePairComparator createTuplePairComparator() {
+ public ITuplePairComparator createTuplePairComparator(IHyracksTaskContext ctx) {
return new JoinComparator(bFactory.createBinaryComparator(), pos0, pos1);
}
}
diff --git a/hyracks/hyracks-hadoop-compat/pom.xml b/hyracks/hyracks-hadoop-compat/pom.xml
index c348dcf..29260ca 100644
--- a/hyracks/hyracks-hadoop-compat/pom.xml
+++ b/hyracks/hyracks-hadoop-compat/pom.xml
@@ -29,7 +29,7 @@
<configuration>
<programs>
<program>
- <mainClass>edu.uci.ics.hyracks.hadoop.compat.CompatibilityLayer</mainClass>
+ <mainClass>edu.uci.ics.hyracks.hadoop.compat.driver.CompatibilityLayer</mainClass>
<name>hadoop-compat</name>
</program>
</programs>
diff --git a/hyracks/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/client/HyracksClient.java b/hyracks/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/client/HyracksClient.java
index eadf679..dfd229a 100644
--- a/hyracks/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/client/HyracksClient.java
+++ b/hyracks/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/client/HyracksClient.java
@@ -66,11 +66,10 @@
JobId jobId;
if (doProfiling) {
System.out.println("PROFILING");
- jobId = connection.createJob(applicationName, spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
+ jobId = connection.startJob(applicationName, spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
} else {
- jobId = connection.createJob(applicationName, spec);
+ jobId = connection.startJob(applicationName, spec);
}
- connection.start(jobId);
HyracksRunningJob runningJob = new HyracksRunningJob(jobId, spec, this);
return runningJob;
}
diff --git a/hyracks/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/client/HyracksRunningJob.java b/hyracks/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/client/HyracksRunningJob.java
index 7c650ec..776fd54 100644
--- a/hyracks/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/client/HyracksRunningJob.java
+++ b/hyracks/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/client/HyracksRunningJob.java
@@ -8,13 +8,13 @@
import org.apache.hadoop.mapred.TaskAttemptID;
import org.apache.hadoop.mapred.TaskCompletionEvent;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
public class HyracksRunningJob implements RunningJob {
JobId jobId;
- JobSpecification spec;
+ IOperatorDescriptorRegistry spec;
HyracksClient hyracksClient;
public JobId getJobId() {
@@ -25,15 +25,15 @@
this.jobId = jobId;
}
- public JobSpecification getSpec() {
+ public IOperatorDescriptorRegistry getSpec() {
return spec;
}
- public void setSpec(JobSpecification spec) {
+ public void setSpec(IOperatorDescriptorRegistry spec) {
this.spec = spec;
}
- public HyracksRunningJob(JobId jobId, JobSpecification jobSpec, HyracksClient hyracksClient) {
+ public HyracksRunningJob(JobId jobId, IOperatorDescriptorRegistry jobSpec, HyracksClient hyracksClient) {
this.spec = jobSpec;
this.jobId = jobId;
this.hyracksClient = hyracksClient;
diff --git a/hyracks/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/util/HadoopAdapter.java b/hyracks/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/util/HadoopAdapter.java
index 28833d7..1d6369a 100644
--- a/hyracks/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/util/HadoopAdapter.java
+++ b/hyracks/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/util/HadoopAdapter.java
@@ -30,6 +30,8 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.hadoop.HadoopMapperOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.hadoop.HadoopReducerOperatorDescriptor;
@@ -171,7 +173,7 @@
}
public HadoopReducerOperatorDescriptor getReducer(JobConf conf,
- JobSpecification spec, boolean useAsCombiner) {
+ IOperatorDescriptorRegistry spec, boolean useAsCombiner) {
HadoopReducerOperatorDescriptor reduceOp = new HadoopReducerOperatorDescriptor(
spec, conf, null, new ClasspathBasedHadoopClassFactory(),
useAsCombiner);
@@ -334,7 +336,7 @@
}
public static InMemorySortOperatorDescriptor getInMemorySorter(
- JobConf conf, JobSpecification spec) {
+ JobConf conf, IOperatorDescriptorRegistry spec) {
InMemorySortOperatorDescriptor inMemorySortOp = null;
RecordDescriptor recordDescriptor = getHadoopRecordDescriptor(conf
.getMapOutputKeyClass().getName(), conf
@@ -352,7 +354,7 @@
}
public static ExternalSortOperatorDescriptor getExternalSorter(
- JobConf conf, JobSpecification spec) {
+ JobConf conf, IOperatorDescriptorRegistry spec) {
ExternalSortOperatorDescriptor externalSortOp = null;
RecordDescriptor recordDescriptor = getHadoopRecordDescriptor(conf
.getMapOutputKeyClass().getName(), conf
@@ -371,7 +373,7 @@
}
public static MToNPartitioningConnectorDescriptor getMtoNHashPartitioningConnector(
- JobConf conf, JobSpecification spec) {
+ JobConf conf, IConnectorDescriptorRegistry spec) {
Class mapOutputKeyClass = conf.getMapOutputKeyClass();
Class mapOutputValueClass = conf.getMapOutputValueClass();
diff --git a/hyracks/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCHandle.java b/hyracks/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCHandle.java
index 749afe2..06b614d 100644
--- a/hyracks/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCHandle.java
+++ b/hyracks/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCHandle.java
@@ -63,8 +63,8 @@
}
@Override
- public synchronized long send(long requestId, Object req, Exception exception) throws IPCException {
- if (state != HandleState.CONNECTED) {
+ public long send(long requestId, Object req, Exception exception) throws IPCException {
+ if (!isConnected()) {
throw new IPCException("Handle is not in Connected state");
}
Message msg = new Message(this);
@@ -131,7 +131,7 @@
setState(HandleState.CLOSED);
}
- synchronized void processIncomingMessages() {
+ void processIncomingMessages() {
inBuffer.flip();
while (Message.hasMessage(inBuffer)) {
Message message = new Message(this);
diff --git a/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/pom.xml b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/pom.xml
new file mode 100644
index 0000000..4a69512
--- /dev/null
+++ b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/pom.xml
@@ -0,0 +1,26 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-virtualcluster-maven-plugin</artifactId>
+ <packaging>maven-plugin</packaging>
+ <name>Hyracks VirtualCluster Maven Plugin</name>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-maven-plugins</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/AbstractHyracksCLIMojo.java b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/AbstractHyracksCLIMojo.java
new file mode 100644
index 0000000..7f3faef
--- /dev/null
+++ b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/AbstractHyracksCLIMojo.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.maven.plugin;
+
+import java.io.File;
+import java.io.PrintWriter;
+
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugin.MojoFailureException;
+
+public abstract class AbstractHyracksCLIMojo extends AbstractHyracksMojo {
+ private static final String HYRACKS_CLI_SCRIPT = "bin" + File.separator + "hyrackscli";
+
+ /**
+ * @parameter
+ * @required
+ */
+ protected File hyracksCLIHome;
+
+ /**
+ * @parameter
+ * @required
+ */
+ private String ccHost;
+
+ /**
+ * @parameter
+ */
+ private int ccPort;
+
+ @Override
+ public void execute() throws MojoExecutionException, MojoFailureException {
+ StringBuilder buffer = new StringBuilder();
+ buffer.append(createConnectCommand());
+ buffer.append('\n');
+ buffer.append(getCommands());
+ final Process proc = launch(new File(hyracksCLIHome, makeScriptName(HYRACKS_CLI_SCRIPT)), null, null);
+ try {
+ PrintWriter out = new PrintWriter(proc.getOutputStream());
+ out.println(buffer.toString());
+ out.close();
+ proc.waitFor();
+ } catch (Exception e) {
+ throw new MojoExecutionException(e.getMessage());
+ }
+ }
+
+ private String createConnectCommand() {
+ return "connect to \"" + ccHost + (ccPort == 0 ? "" : (":" + ccPort)) + "\";";
+ }
+
+ protected abstract String getCommands();
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/AbstractHyracksMojo.java b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/AbstractHyracksMojo.java
new file mode 100644
index 0000000..f35c78a
--- /dev/null
+++ b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/AbstractHyracksMojo.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.maven.plugin;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+
+import org.apache.maven.plugin.AbstractMojo;
+import org.apache.maven.plugin.MojoExecutionException;
+
+public abstract class AbstractHyracksMojo extends AbstractMojo {
+ /**
+ * @parameter
+ */
+ protected String jvmOptions;
+
+ protected Process launch(File command, String options, File workingDir) throws MojoExecutionException {
+ if (!command.isFile()) {
+ throw new MojoExecutionException(command.getAbsolutePath() + " is not an executable program");
+ }
+
+ getLog().info("Executing Hyracks command: " + command + " with args [" + options + "]");
+ String osName = System.getProperty("os.name");
+ try {
+ if (osName.startsWith("Windows")) {
+ return launchWindowsBatch(command, options);
+ } else {
+ return launchUnixScript(command, options, workingDir);
+ }
+ } catch (IOException e) {
+ throw new MojoExecutionException("Error executing command: " + command.getAbsolutePath(), e);
+ }
+ }
+
+ protected Process launchWindowsBatch(File command, String options) throws IOException {
+ String[] commandWithOptions = new String[] { "cmd.exe", "/C", command.getAbsolutePath() + " " + options };
+
+ Process proc = Runtime.getRuntime().exec(commandWithOptions);
+ dump(proc.getInputStream());
+ dump(proc.getErrorStream());
+ return proc;
+ }
+
+ protected Process launchUnixScript(File command, String options, File workingDir) throws IOException {
+ String[] optionsArray = new String[0];
+ if (options != null && !options.trim().isEmpty()) {
+ optionsArray = options.trim().split("\\s+");
+ }
+ String[] commandWithOptions = new String[optionsArray.length + 1];
+ commandWithOptions[0] = command.getAbsolutePath();
+ for (int i = 0; i < optionsArray.length; ++i) {
+ commandWithOptions[i + 1] = optionsArray[i];
+ }
+ ProcessBuilder pb = new ProcessBuilder(commandWithOptions);
+ if (jvmOptions != null) {
+ pb.environment().put("JAVA_OPTS", jvmOptions);
+ }
+ pb.directory(workingDir == null ? new File(".") : workingDir);
+ Process proc = pb.start();
+ dump(proc.getInputStream());
+ dump(proc.getErrorStream());
+ return proc;
+ }
+
+ protected void dump(final InputStream input) {
+ final int streamBufferSize = 1000;
+ final Reader in = new InputStreamReader(input);
+ new Thread(new Runnable() {
+ public void run() {
+ try {
+ char[] chars = new char[streamBufferSize];
+ int c;
+ while ((c = in.read(chars)) != -1) {
+ if (c > 0) {
+ System.out.print(String.valueOf(chars, 0, c));
+ }
+ }
+ } catch (IOException e) {
+ }
+ }
+ }).start();
+ }
+
+ protected String makeScriptName(String scriptName) {
+ String osName = System.getProperty("os.name");
+ String commandExt = osName.startsWith("Windows") ? ".bat" : "";
+ return scriptName + commandExt;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IReceiverEventListener.java b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/AbstractHyracksServerMojo.java
similarity index 67%
copy from hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IReceiverEventListener.java
copy to hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/AbstractHyracksServerMojo.java
index 98c083e..448ca22 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IReceiverEventListener.java
+++ b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/AbstractHyracksServerMojo.java
@@ -12,10 +12,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.api.comm;
+package edu.uci.ics.hyracks.maven.plugin;
-public interface IReceiverEventListener {
- public void notifySenderAvailability(int sender);
+import java.io.File;
- public void notifySenderCount(int senderCount);
+public abstract class AbstractHyracksServerMojo extends AbstractHyracksMojo {
+ /**
+ * @parameter
+ * @required
+ */
+ protected File hyracksServerHome;
+
+ /**
+ * @parameter
+ */
+ protected File workingDir;
}
\ No newline at end of file
diff --git a/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/HyracksAppDeploymentMojo.java b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/HyracksAppDeploymentMojo.java
new file mode 100644
index 0000000..76bbb5a
--- /dev/null
+++ b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/HyracksAppDeploymentMojo.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.maven.plugin;
+
+import java.io.File;
+
+/**
+ * @goal deploy-app
+ */
+public class HyracksAppDeploymentMojo extends AbstractHyracksCLIMojo {
+ /**
+ * @parameter
+ * @required
+ */
+ private String appName;
+
+ /**
+ * @parameter
+ * @required
+ */
+ private File harFile;
+
+ @Override
+ protected String getCommands() {
+ return "create application " + appName + " \"" + harFile.getAbsolutePath() + "\";";
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/HyracksCCStartMojo.java b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/HyracksCCStartMojo.java
new file mode 100644
index 0000000..cf78a72
--- /dev/null
+++ b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/HyracksCCStartMojo.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.maven.plugin;
+
+import java.io.File;
+
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugin.MojoFailureException;
+
+/**
+ * @goal start-cc
+ */
+public class HyracksCCStartMojo extends AbstractHyracksServerMojo {
+ private static final String HYRACKS_CC_SCRIPT = "bin" + File.separator + "hyrackscc";
+
+ /**
+ * @parameter property = "port"
+ */
+ private int port;
+
+ @Override
+ public void execute() throws MojoExecutionException, MojoFailureException {
+ StringBuilder cmdLineBuffer = new StringBuilder();
+ if (port != 0) {
+ cmdLineBuffer.append("-port ").append(port);
+ }
+ cmdLineBuffer.append(" -client-net-ip-address 127.0.0.1");
+ cmdLineBuffer.append(" -cluster-net-ip-address 127.0.0.1");
+ String args = cmdLineBuffer.toString();
+ final Process proc = launch(new File(hyracksServerHome, makeScriptName(HYRACKS_CC_SCRIPT)), args, workingDir);
+ HyracksServiceRegistry.INSTANCE.addServiceProcess(proc);
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/HyracksNCStartMojo.java b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/HyracksNCStartMojo.java
new file mode 100644
index 0000000..47de024
--- /dev/null
+++ b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/HyracksNCStartMojo.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.maven.plugin;
+
+import java.io.File;
+
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugin.MojoFailureException;
+
+/**
+ * @goal start-nc
+ */
+public class HyracksNCStartMojo extends AbstractHyracksServerMojo {
+ private static final String HYRACKS_NC_SCRIPT = "bin" + File.separator + "hyracksnc";
+
+ /**
+ * @parameter
+ * @required
+ */
+ private String nodeId;
+
+ /**
+ * @parameter
+ * @required
+ */
+ private String ccHost;
+
+ /**
+ * @parameter
+ */
+ private int ccPort;
+
+ /**
+ * @parameter
+ * @required
+ */
+ private String dataIpAddress;
+
+ @Override
+ public void execute() throws MojoExecutionException, MojoFailureException {
+ StringBuilder cmdLineBuffer = new StringBuilder();
+ cmdLineBuffer.append(" -cc-host ").append(ccHost);
+ cmdLineBuffer.append(" -data-ip-address ").append(dataIpAddress);
+ cmdLineBuffer.append(" -node-id ").append(nodeId);
+ cmdLineBuffer.append(" -cluster-net-ip-address 127.0.0.1");
+ if (ccPort != 0) {
+ cmdLineBuffer.append(" -cc-port ").append(ccPort);
+ }
+ String args = cmdLineBuffer.toString();
+ final Process proc = launch(new File(hyracksServerHome, makeScriptName(HYRACKS_NC_SCRIPT)), args, workingDir);
+ HyracksServiceRegistry.INSTANCE.addServiceProcess(proc);
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/HyracksServiceRegistry.java b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/HyracksServiceRegistry.java
new file mode 100644
index 0000000..6ea5a64
--- /dev/null
+++ b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/HyracksServiceRegistry.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.maven.plugin;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class HyracksServiceRegistry {
+ public static HyracksServiceRegistry INSTANCE = new HyracksServiceRegistry();
+
+ private final List<Process> serviceProcesses;
+
+ private HyracksServiceRegistry() {
+ serviceProcesses = new ArrayList<Process>();
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ destroyAll();
+ }
+ });
+ }
+
+ public synchronized void addServiceProcess(Process process) {
+ serviceProcesses.add(process);
+ }
+
+ public synchronized void destroyAll() {
+ for (Process p : serviceProcesses) {
+ try {
+ p.destroy();
+ } catch (Exception e) {
+ }
+ }
+ serviceProcesses.clear();
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/HyracksStopServicesMojo.java b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/HyracksStopServicesMojo.java
new file mode 100644
index 0000000..eaf40fc
--- /dev/null
+++ b/hyracks/hyracks-maven-plugins/hyracks-virtualcluster-maven-plugin/src/main/java/edu/uci/ics/hyracks/maven/plugin/HyracksStopServicesMojo.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.maven.plugin;
+
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugin.MojoFailureException;
+
+/**
+ * @goal stop-services
+ */
+public class HyracksStopServicesMojo extends AbstractHyracksMojo {
+ @Override
+ public void execute() throws MojoExecutionException, MojoFailureException {
+ getLog().info("Stopping Hyracks Services");
+ HyracksServiceRegistry.INSTANCE.destroyAll();
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-maven-plugins/pom.xml b/hyracks/hyracks-maven-plugins/pom.xml
new file mode 100644
index 0000000..679896d
--- /dev/null
+++ b/hyracks/hyracks-maven-plugins/pom.xml
@@ -0,0 +1,25 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-maven-plugins</artifactId>
+ <packaging>pom</packaging>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </parent>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-plugin-api</artifactId>
+ <version>2.2.1</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+
+ <modules>
+ <module>hyracks-virtualcluster-maven-plugin</module>
+ </modules>
+</project>
diff --git a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/IBufferAcceptor.java b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/IBufferAcceptor.java
index 2f27bf0..26e335e 100644
--- a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/IBufferAcceptor.java
+++ b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/IBufferAcceptor.java
@@ -16,6 +16,16 @@
import java.nio.ByteBuffer;
+/**
+ * Accepts buffers.
+ *
+ * @author vinayakb
+ */
public interface IBufferAcceptor {
+ /**
+ * Accept a buffer.
+ *
+ * @param buffer
+ */
public void accept(ByteBuffer buffer);
}
\ No newline at end of file
diff --git a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/ICloseableBufferAcceptor.java b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/ICloseableBufferAcceptor.java
index c395ac9..5e9bf02 100644
--- a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/ICloseableBufferAcceptor.java
+++ b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/ICloseableBufferAcceptor.java
@@ -14,8 +14,23 @@
*/
package edu.uci.ics.hyracks.net.buffers;
+/**
+ * A buffer acceptor that can be closed to indicate end of transmission or an error code
+ * specified to indicate an error in transmission.
+ *
+ * @author vinayakb
+ */
public interface ICloseableBufferAcceptor extends IBufferAcceptor {
+ /**
+ * Close the buffer acceptor.
+ */
public void close();
+ /**
+ * Indicate that an error occurred.
+ *
+ * @param ecode
+ * - the error code.
+ */
public void error(int ecode);
}
\ No newline at end of file
diff --git a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelControlBlock.java b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
index 4b55d4b..3dec7e0 100644
--- a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
+++ b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
@@ -28,6 +28,11 @@
import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
import edu.uci.ics.hyracks.net.exceptions.NetException;
+/**
+ * Handle to a channel that represents a logical full-duplex communication end-point.
+ *
+ * @author vinayakb
+ */
public class ChannelControlBlock {
private static final Logger LOGGER = Logger.getLogger(ChannelControlBlock.class.getName());
@@ -45,6 +50,8 @@
private final AtomicBoolean remoteClose;
+ private final AtomicBoolean remoteCloseAck;
+
ChannelControlBlock(ChannelSet cSet, int channelId) {
this.cSet = cSet;
this.channelId = channelId;
@@ -53,16 +60,27 @@
localClose = new AtomicBoolean();
localCloseAck = new AtomicBoolean();
remoteClose = new AtomicBoolean();
+ remoteCloseAck = new AtomicBoolean();
}
int getChannelId() {
return channelId;
}
+ /**
+ * Get the read inderface of this channel.
+ *
+ * @return the read interface.
+ */
public IChannelReadInterface getReadInterface() {
return ri;
}
+ /**
+ * Get the write interface of this channel.
+ *
+ * @return the write interface.
+ */
public IChannelWriteInterface getWriteInterface() {
return wi;
}
@@ -110,21 +128,21 @@
if (size <= 0) {
return size;
}
- if (ri.currentReadBuffer == null) {
- ri.currentReadBuffer = ri.riEmptyStack.poll();
- assert ri.currentReadBuffer != null;
+ if (currentReadBuffer == null) {
+ currentReadBuffer = riEmptyStack.poll();
+ assert currentReadBuffer != null;
}
- int rSize = Math.min(size, ri.currentReadBuffer.remaining());
+ int rSize = Math.min(size, currentReadBuffer.remaining());
if (rSize > 0) {
- ri.currentReadBuffer.limit(ri.currentReadBuffer.position() + rSize);
+ currentReadBuffer.limit(currentReadBuffer.position() + rSize);
int len;
try {
- len = sc.read(ri.currentReadBuffer);
+ len = sc.read(currentReadBuffer);
if (len < 0) {
throw new NetException("Socket Closed");
}
} finally {
- ri.currentReadBuffer.limit(ri.currentReadBuffer.capacity());
+ currentReadBuffer.limit(currentReadBuffer.capacity());
}
size -= len;
if (len < rSize) {
@@ -133,7 +151,7 @@
} else {
return size;
}
- if (ri.currentReadBuffer.remaining() <= 0) {
+ if (currentReadBuffer.remaining() <= 0) {
flush();
}
}
@@ -142,7 +160,7 @@
void flush() {
if (currentReadBuffer != null) {
currentReadBuffer.flip();
- fba.accept(ri.currentReadBuffer);
+ fba.accept(currentReadBuffer);
currentReadBuffer = null;
}
}
@@ -241,7 +259,7 @@
ecodeSent = true;
localClose.set(true);
adjustChannelWritability();
- } else if (wi.eos && !wi.eosSent) {
+ } else if (eos && !eosSent) {
writerState.command.setChannelId(channelId);
writerState.command.setCommandType(MuxDemuxCommand.CommandType.CLOSE_CHANNEL);
writerState.command.setData(0);
@@ -321,11 +339,15 @@
remoteClose.set(true);
}
+ void reportRemoteEOSAck() {
+ remoteCloseAck.set(true);
+ }
+
boolean getRemoteEOS() {
return remoteClose.get();
}
- synchronized void reportLocalEOSAck() {
+ void reportLocalEOSAck() {
localCloseAck.set(true);
}
@@ -336,12 +358,13 @@
}
boolean completelyClosed() {
- return localCloseAck.get() && remoteClose.get();
+ return localCloseAck.get() && remoteCloseAck.get();
}
@Override
public String toString() {
return "Channel:" + channelId + "[localClose: " + localClose + " localCloseAck: " + localCloseAck
- + " remoteClose: " + remoteClose + " readCredits: " + ri.credits + " writeCredits: " + wi.credits + "]";
+ + " remoteClose: " + remoteClose + " remoteCloseAck:" + remoteCloseAck + " readCredits: " + ri.credits
+ + " writeCredits: " + wi.credits + "]";
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelSet.java b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelSet.java
index ffdb5c6..dabc8a4 100644
--- a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelSet.java
+++ b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelSet.java
@@ -24,8 +24,6 @@
public class ChannelSet {
private static final Logger LOGGER = Logger.getLogger(ChannelSet.class.getName());
- private static final int MAX_OPEN_CHANNELS = 1024;
-
private static final int INITIAL_SIZE = 16;
private final MultiplexedConnection mConn;
@@ -61,7 +59,7 @@
ChannelControlBlock allocateChannel() throws NetException {
synchronized (mConn) {
int idx = allocationBitmap.nextClearBit(0);
- if (idx < 0 || idx == ccbArray.length) {
+ if (idx < 0 || idx >= ccbArray.length) {
cleanupClosedChannels();
idx = allocationBitmap.nextClearBit(0);
if (idx < 0 || idx == ccbArray.length) {
@@ -183,21 +181,35 @@
void markEOSAck(int channelId) {
synchronized (mConn) {
- assert !pendingEOSAckBitmap.get(channelId);
- pendingEOSAckBitmap.set(channelId);
- pendingWriteEventsCounter.increment();
+ if (!pendingEOSAckBitmap.get(channelId)) {
+ pendingEOSAckBitmap.set(channelId);
+ pendingWriteEventsCounter.increment();
+ }
+ }
+ }
+
+ void notifyIOError() {
+ synchronized (mConn) {
+ for (int i = 0; i < ccbArray.length; ++i) {
+ ChannelControlBlock ccb = ccbArray[i];
+ if (ccb != null && !ccb.getRemoteEOS()) {
+ ccb.reportRemoteError(-1);
+ markEOSAck(i);
+ unmarkPendingCredits(i);
+ }
+ }
}
}
private ChannelControlBlock createChannel(int idx) throws NetException {
+ if (idx > MuxDemuxCommand.MAX_CHANNEL_ID) {
+ throw new NetException("Channel Id > " + MuxDemuxCommand.MAX_CHANNEL_ID + " being opened");
+ }
if (idx >= ccbArray.length) {
expand(idx);
}
- if (idx > MAX_OPEN_CHANNELS) {
- throw new NetException("More than " + MAX_OPEN_CHANNELS + " opened concurrently");
- }
if (ccbArray[idx] != null) {
- assert ccbArray[idx].completelyClosed();
+ assert ccbArray[idx].completelyClosed() : ccbArray[idx].toString();
if (ccbArray[idx].completelyClosed()) {
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("Cleaning free channel: " + ccbArray[idx]);
diff --git a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelOpenListener.java b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelOpenListener.java
index 0fc9b2a..feb605f 100644
--- a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelOpenListener.java
+++ b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelOpenListener.java
@@ -14,6 +14,17 @@
*/
package edu.uci.ics.hyracks.net.protocols.muxdemux;
+/**
+ * Callback interface to report opening of channels.
+ *
+ * @author vinayakb
+ */
public interface IChannelOpenListener {
+ /**
+ * Indicates that a remote endpoint has opened a channel to this receiver.
+ *
+ * @param channel
+ * - The newly opened channel.
+ */
public void channelOpened(ChannelControlBlock channel);
}
\ No newline at end of file
diff --git a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelReadInterface.java b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelReadInterface.java
index 468a617..178ad78 100644
--- a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelReadInterface.java
+++ b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelReadInterface.java
@@ -17,8 +17,26 @@
import edu.uci.ics.hyracks.net.buffers.IBufferAcceptor;
import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
+/**
+ * Represents the read interface of a {@link ChannelControlBlock}.
+ *
+ * @author vinayakb
+ */
public interface IChannelReadInterface {
+ /**
+ * Set the callback that will be invoked by the network layer when a buffer has been
+ * filled with data received from the remote side.
+ *
+ * @param fullBufferAcceptor
+ * - the full buffer acceptor.
+ */
public void setFullBufferAcceptor(ICloseableBufferAcceptor fullBufferAcceptor);
+ /**
+ * Get the acceptor that collects empty buffers when the client has finished consuming
+ * a previously full buffer.
+ *
+ * @return the empty buffer acceptor.
+ */
public IBufferAcceptor getEmptyBufferAcceptor();
}
\ No newline at end of file
diff --git a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelWriteInterface.java b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelWriteInterface.java
index 1e53d71..8e71246 100644
--- a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelWriteInterface.java
+++ b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelWriteInterface.java
@@ -17,8 +17,26 @@
import edu.uci.ics.hyracks.net.buffers.IBufferAcceptor;
import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
+/**
+ * Represents the write interface of a {@link ChannelControlBlock}.
+ *
+ * @author vinayakb
+ */
public interface IChannelWriteInterface {
+ /**
+ * Set the callback interface that must be invoked when a full buffer has been emptied by
+ * writing the data to the remote end.
+ *
+ * @param emptyBufferAcceptor
+ * - the empty buffer acceptor.
+ */
public void setEmptyBufferAcceptor(IBufferAcceptor emptyBufferAcceptor);
+ /**
+ * Get the full buffer acceptor that accepts buffers filled with data that need to be written
+ * to the remote end.
+ *
+ * @return the full buffer acceptor.
+ */
public ICloseableBufferAcceptor getFullBufferAcceptor();
}
\ No newline at end of file
diff --git a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MultiplexedConnection.java b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MultiplexedConnection.java
index c905f57..27bd793 100644
--- a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MultiplexedConnection.java
+++ b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MultiplexedConnection.java
@@ -26,6 +26,12 @@
import edu.uci.ics.hyracks.net.protocols.tcp.ITCPConnectionEventListener;
import edu.uci.ics.hyracks.net.protocols.tcp.TCPConnection;
+/**
+ * A {@link MultiplexedConnection} can be used by clients to create multiple "channels"
+ * that can have independent full-duplex conversations.
+ *
+ * @author vinayakb
+ */
public class MultiplexedConnection implements ITCPConnectionEventListener {
private static final Logger LOGGER = Logger.getLogger(MultiplexedConnection.class.getName());
@@ -49,7 +55,9 @@
private boolean connectionFailure;
- public MultiplexedConnection(MuxDemux muxDemux) {
+ private Exception error;
+
+ MultiplexedConnection(MuxDemux muxDemux) {
this.muxDemux = muxDemux;
pendingWriteEventsCounter = new IEventCounter() {
private int counter;
@@ -119,7 +127,26 @@
}
}
- public ChannelControlBlock openChannel() throws NetException, InterruptedException {
+ @Override
+ public synchronized void notifyIOError(Exception e) {
+ connectionFailure = true;
+ error = e;
+ cSet.notifyIOError();
+ }
+
+ /**
+ * Open a channel to the other side.
+ *
+ * @return
+ * @throws NetException
+ * - A network failure occurred.
+ */
+ public ChannelControlBlock openChannel() throws NetException {
+ synchronized (this) {
+ if (connectionFailure) {
+ throw new NetException(error);
+ }
+ }
ChannelControlBlock channel = cSet.allocateChannel();
int channelId = channel.getChannelId();
cSet.initiateChannelSyn(channelId);
@@ -127,7 +154,7 @@
}
class WriterState {
- private final ByteBuffer writeBuffer;
+ private final ByteBuffer cmdWriteBuffer;
final MuxDemuxCommand command;
@@ -138,29 +165,29 @@
private ChannelControlBlock ccb;
public WriterState() {
- writeBuffer = ByteBuffer.allocateDirect(MuxDemuxCommand.COMMAND_SIZE);
- writeBuffer.flip();
+ cmdWriteBuffer = ByteBuffer.allocateDirect(MuxDemuxCommand.COMMAND_SIZE);
+ cmdWriteBuffer.flip();
command = new MuxDemuxCommand();
ccb = null;
}
boolean writePending() {
- return writeBuffer.remaining() > 0 || (pendingBuffer != null && pendingWriteSize > 0);
+ return cmdWriteBuffer.remaining() > 0 || (pendingBuffer != null && pendingWriteSize > 0);
}
void reset(ByteBuffer pendingBuffer, int pendingWriteSize, ChannelControlBlock ccb) {
- writeBuffer.clear();
- command.write(writeBuffer);
- writeBuffer.flip();
+ cmdWriteBuffer.clear();
+ command.write(cmdWriteBuffer);
+ cmdWriteBuffer.flip();
this.pendingBuffer = pendingBuffer;
this.pendingWriteSize = pendingWriteSize;
this.ccb = ccb;
}
boolean performPendingWrite(SocketChannel sc) throws IOException {
- int len = writeBuffer.remaining();
+ int len = cmdWriteBuffer.remaining();
if (len > 0) {
- int written = sc.write(writeBuffer);
+ int written = sc.write(cmdWriteBuffer);
muxDemux.getPerformanceCounters().addSignalingBytesWritten(written);
if (written < len) {
return false;
@@ -250,6 +277,8 @@
BitSet pendingEOSAckBitmap = cSet.getPendingEOSAckBitmap();
for (int j = pendingEOSAckBitmap.nextSetBit(0); j >= 0; j = pendingEOSAckBitmap.nextSetBit(j)) {
pendingEOSAckBitmap.clear(j);
+ ChannelControlBlock ccb = cSet.getCCB(j);
+ ccb.reportRemoteEOSAck();
writerState.command.setChannelId(j);
writerState.command.setCommandType(MuxDemuxCommand.CommandType.CLOSE_CHANNEL_ACK);
writerState.command.setData(0);
@@ -261,9 +290,9 @@
}
BitSet pendingChannelWriteBitmap = cSet.getPendingChannelWriteBitmap();
lastChannelWritten = pendingChannelWriteBitmap.nextSetBit(lastChannelWritten + 1);
- if (lastChannelWritten < 0) {
+ if (lastChannelWritten == -1) {
lastChannelWritten = pendingChannelWriteBitmap.nextSetBit(0);
- if (lastChannelWritten < 0) {
+ if (lastChannelWritten == -1) {
return;
}
}
diff --git a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemux.java b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemux.java
index 8548bb8..c719bc4 100644
--- a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemux.java
+++ b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemux.java
@@ -24,20 +24,43 @@
import edu.uci.ics.hyracks.net.protocols.tcp.TCPConnection;
import edu.uci.ics.hyracks.net.protocols.tcp.TCPEndpoint;
+/**
+ * Multiplexed Connection Manager.
+ * Every participant that wants to use the multiplexed connections must create and instance
+ * of this class.
+ *
+ * @author vinayakb
+ */
public class MuxDemux {
private final InetSocketAddress localAddress;
private final IChannelOpenListener channelOpenListener;
+ private final int maxConnectionAttempts;
+
private final Map<InetSocketAddress, MultiplexedConnection> connectionMap;
private final TCPEndpoint tcpEndpoint;
private final MuxDemuxPerformanceCounters perfCounters;
- public MuxDemux(InetSocketAddress localAddress, IChannelOpenListener listener, int nThreads) {
+ /**
+ * Constructor.
+ *
+ * @param localAddress
+ * - TCP/IP socket address to listen on
+ * @param listener
+ * - Callback interface to report channel events
+ * @param nThreads
+ * - Number of threads to use for data transfer
+ * @param maxConnectionAttempts
+ * - Maximum number of connection attempts
+ */
+ public MuxDemux(InetSocketAddress localAddress, IChannelOpenListener listener, int nThreads,
+ int maxConnectionAttempts) {
this.localAddress = localAddress;
this.channelOpenListener = listener;
+ this.maxConnectionAttempts = maxConnectionAttempts;
connectionMap = new HashMap<InetSocketAddress, MultiplexedConnection>();
this.tcpEndpoint = new TCPEndpoint(new ITCPConnectionListener() {
@Override
@@ -67,7 +90,7 @@
mConn = connectionMap.get(remoteAddress);
assert mConn != null;
int nConnectionAttempts = mConn.getConnectionAttempts();
- if (nConnectionAttempts > 5) {
+ if (nConnectionAttempts > MuxDemux.this.maxConnectionAttempts) {
connectionMap.remove(remoteAddress);
mConn.setConnectionFailure();
} else {
@@ -80,10 +103,27 @@
perfCounters = new MuxDemuxPerformanceCounters();
}
+ /**
+ * Starts listening for remote connections and is capable of initiating connections.
+ *
+ * @throws IOException
+ */
public void start() throws IOException {
tcpEndpoint.start(localAddress);
}
+ /**
+ * Create a {@link MultiplexedConnection} that can create channels to the specified remote address.
+ * The remote address must have a {@link MuxDemux} listening for connections.
+ *
+ * @param remoteAddress
+ * - Address of the remote {@link MuxDemux}
+ * @return a {@link MultiplexedConnection} connected to the remote address.
+ * @throws InterruptedException
+ * - This call was interrupted while waiting for connection setup.
+ * In such an event, it is safe to retry the {@link #connect(InetSocketAddress)} call.
+ * @throws NetException
+ */
public MultiplexedConnection connect(InetSocketAddress remoteAddress) throws InterruptedException, NetException {
MultiplexedConnection mConn = null;
synchronized (this) {
@@ -102,10 +142,20 @@
return channelOpenListener;
}
+ /**
+ * Get the local address that this {@link MuxDemux} is listening for connections.
+ *
+ * @return local TCP/IP socket address.
+ */
public InetSocketAddress getLocalAddress() {
return tcpEndpoint.getLocalAddress();
}
+ /**
+ * Gets performance counters useful for collecting efficiency metrics.
+ *
+ * @return
+ */
public MuxDemuxPerformanceCounters getPerformanceCounters() {
return perfCounters;
}
diff --git a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemuxCommand.java b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemuxCommand.java
index 2e2636b..4610170 100644
--- a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemuxCommand.java
+++ b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemuxCommand.java
@@ -5,11 +5,11 @@
import edu.uci.ics.hyracks.net.exceptions.NetException;
class MuxDemuxCommand {
- static final int MAX_CHANNEL_ID = 0x3ff;
+ static final int MAX_CHANNEL_ID = Integer.MAX_VALUE - 1;
- static final int COMMAND_SIZE = 4;
+ static final int COMMAND_SIZE = 8;
- static final int MAX_DATA_VALUE = 0x7ffff;
+ static final int MAX_DATA_VALUE = 0x1fffffff;
enum CommandType {
OPEN_CHANNEL,
@@ -57,15 +57,15 @@
}
public void write(ByteBuffer buffer) {
- int cmd = (channelId << 22) | (type.ordinal() << 19) | (data & 0x7ffff);
- buffer.putInt(cmd);
+ long cmd = (((long) channelId) << 32) | (((long) type.ordinal()) << 29) | (data & 0x1fffffff);
+ buffer.putLong(cmd);
}
public void read(ByteBuffer buffer) {
- int cmd = buffer.getInt();
- channelId = (cmd >> 22) & 0x3ff;
- type = CommandType.values()[(cmd >> 19) & 0x7];
- data = cmd & 0x7ffff;
+ long cmd = buffer.getLong();
+ channelId = (int) ((cmd >> 32) & 0x7fffffff);
+ type = CommandType.values()[(int) ((cmd >> 29) & 0x7)];
+ data = (int) (cmd & 0x1fffffff);
}
@Override
diff --git a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/ITCPConnectionEventListener.java b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/ITCPConnectionEventListener.java
index 607bf31..79c21ee 100644
--- a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/ITCPConnectionEventListener.java
+++ b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/ITCPConnectionEventListener.java
@@ -21,4 +21,6 @@
public interface ITCPConnectionEventListener {
public void notifyIOReady(TCPConnection connection, boolean readable, boolean writable) throws IOException,
NetException;
+
+ public void notifyIOError(Exception e);
}
\ No newline at end of file
diff --git a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/TCPEndpoint.java b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/TCPEndpoint.java
index 1a73bdc..d13a17e 100644
--- a/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/TCPEndpoint.java
+++ b/hyracks/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/TCPEndpoint.java
@@ -88,10 +88,11 @@
private final List<SocketChannel> workingIncomingConnections;
- private Selector selector;
+ private final Selector selector;
public IOThread() throws IOException {
super("TCPEndpoint IO Thread");
+ setDaemon(true);
setPriority(MAX_PRIORITY);
this.pendingConnections = new ArrayList<InetSocketAddress>();
this.workingPendingConnections = new ArrayList<InetSocketAddress>();
@@ -116,7 +117,9 @@
connect = channel.connect(address);
} catch (IOException e) {
failure = true;
- connectionListener.connectionFailure(address);
+ synchronized (connectionListener) {
+ connectionListener.connectionFailure(address);
+ }
}
if (!failure) {
if (!connect) {
@@ -153,7 +156,13 @@
if (readable || writable) {
TCPConnection connection = (TCPConnection) key.attachment();
- connection.getEventListener().notifyIOReady(connection, readable, writable);
+ try {
+ connection.getEventListener().notifyIOReady(connection, readable, writable);
+ } catch (Exception e) {
+ connection.getEventListener().notifyIOError(e);
+ connection.close();
+ continue;
+ }
}
if (key.isAcceptable()) {
assert sc == serverSocketChannel;
@@ -167,7 +176,9 @@
} catch (Exception e) {
e.printStackTrace();
key.cancel();
- connectionListener.connectionFailure((InetSocketAddress) key.attachment());
+ synchronized (connectionListener) {
+ connectionListener.connectionFailure((InetSocketAddress) key.attachment());
+ }
}
if (finishConnect) {
createConnection(key, channel);
@@ -185,7 +196,9 @@
TCPConnection connection = new TCPConnection(TCPEndpoint.this, channel, key, selector);
key.attach(connection);
key.interestOps(0);
- connectionListener.connectionEstablished(connection);
+ synchronized (connectionListener) {
+ connectionListener.connectionEstablished(connection);
+ }
}
synchronized void initiateConnection(InetSocketAddress remoteAddress) {
diff --git a/hyracks/hyracks-net/src/test/java/edu/uci/ics/hyracks/net/tests/NetTest.java b/hyracks/hyracks-net/src/test/java/edu/uci/ics/hyracks/net/tests/NetTest.java
index 31bc2df..5002b81 100644
--- a/hyracks/hyracks-net/src/test/java/edu/uci/ics/hyracks/net/tests/NetTest.java
+++ b/hyracks/hyracks-net/src/test/java/edu/uci/ics/hyracks/net/tests/NetTest.java
@@ -158,7 +158,7 @@
}.start();
}
};
- return new MuxDemux(new InetSocketAddress("127.0.0.1", 0), md1OpenListener, 1);
+ return new MuxDemux(new InetSocketAddress("127.0.0.1", 0), md1OpenListener, 1, 5);
}
private class ChannelIO {
diff --git a/hyracks/hyracks-storage-am-btree/pom.xml b/hyracks/hyracks-storage-am-btree/pom.xml
index 4cdd094..dc38218 100644
--- a/hyracks/hyracks-storage-am-btree/pom.xml
+++ b/hyracks/hyracks-storage-am-btree/pom.xml
@@ -53,13 +53,6 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>edu.uci.ics.hyracks</groupId>
- <artifactId>hyracks-control-nc</artifactId>
- <version>0.2.1-SNAPSHOT</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.8.1</version>
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeFrame.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeFrame.java
index a24c4d7..7a61d09 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeFrame.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeFrame.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.btree.api;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
@@ -7,7 +22,7 @@
public interface IBTreeFrame extends ITreeIndexFrame {
public int findUpdateTupleIndex(ITupleReference tuple) throws TreeIndexException;
- public int findInsertTupleIndex(ITupleReference tuple) throws TreeIndexException;
+ public int findInsertTupleIndex(ITupleReference tuple) throws TreeIndexException;
public int findDeleteTupleIndex(ITupleReference tuple) throws TreeIndexException;
public void insertSorted(ITupleReference tuple);
public boolean getSmFlag();
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeLeafFrame.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
index a079527..74bf2b0 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
@@ -18,6 +18,7 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleMode;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
@@ -27,10 +28,9 @@
public int getNextLeaf();
- public void setPrevLeaf(int prevPage);
-
- public int getPrevLeaf();
-
public int findTupleIndex(ITupleReference searchKey, ITreeIndexTupleReference pageTuple, MultiComparator cmp,
FindTupleMode ftm, FindTupleNoExactMatchPolicy ftp) throws HyracksDataException;
+
+ public int findUpsertTupleIndex(ITupleReference tuple) throws TreeIndexException;
+ public ITupleReference getUpsertBeforeTuple(ITupleReference tuple, int targetTupleIndex) throws TreeIndexException;
}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java
index 4486205..f78b6e4 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java
@@ -247,7 +247,7 @@
// buf.getInt(prevRec.getFieldOff()) + " " +
// buf.getInt(prevRec.getFieldOff()+4));
prefixFreeSpace += tupleWriter.writeTupleFields(prevTuple, 0, fieldCountToCompress,
- byteBuffer, prefixFreeSpace);
+ byteBuffer.array(), prefixFreeSpace);
// System.out.println("WRITING PREFIX RECORD " +
// prefixSlotNum + " AT " + tmp + " " +
// freeSpace);
@@ -265,7 +265,7 @@
newTupleSlots[tupleCount - 1 - currTupleIndex] = slotManager.encodeSlotFields(
prefixTupleIndex, tupleFreeSpace);
tupleFreeSpace += tupleWriter.writeTupleFields(tupleToWrite, fieldCountToCompress,
- fieldCount - fieldCountToCompress, byteBuffer, tupleFreeSpace);
+ fieldCount - fieldCountToCompress, byteBuffer.array(), tupleFreeSpace);
}
prefixTupleIndex++;
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
index 3e235b3..699915d 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
@@ -17,32 +17,26 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.common.util.IndexUtils;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
public class BTreeDataflowHelper extends TreeIndexDataflowHelper {
- public BTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- boolean createIfNotExists) {
- super(opDesc, ctx, partition, createIfNotExists);
+ public BTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition) {
+ super(opDesc, ctx, partition);
}
-
+
@Override
public ITreeIndex createIndexInstance() throws HyracksDataException {
- MultiComparator cmp = IndexUtils.createMultiComparator(treeOpDesc.getTreeIndexComparatorFactories());
- IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
- ITreeIndexMetaDataFrameFactory metaDataFrameFactory = new LIFOMetaDataFrameFactory();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, indexFileId, 0,
- metaDataFrameFactory);
- return new BTree(bufferCache, treeOpDesc.getTreeIndexTypeTraits().length, cmp, freePageManager,
- treeOpDesc.getTreeIndexInteriorFactory(), treeOpDesc.getTreeIndexLeafFactory());
+ try {
+ return BTreeUtils.createBTree(opDesc.getStorageManager().getBufferCache(ctx), opDesc
+ .getOpCallbackProvider().getOperationCallback(), treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc
+ .getTreeIndexComparatorFactories(), BTreeLeafFrameType.REGULAR_NSM);
+ } catch (BTreeException e) {
+ throw new HyracksDataException(e);
+ }
}
}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java
index 308e305..57d7564 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java
@@ -26,7 +26,7 @@
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition, boolean createIfNotExists) {
- return new BTreeDataflowHelper(opDesc, ctx, partition, createIfNotExists);
+ int partition) {
+ return new BTreeDataflowHelper(opDesc, ctx, partition);
}
}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
index 6314260..f403312 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
@@ -21,9 +21,9 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
@@ -34,22 +34,21 @@
private static final long serialVersionUID = 1L;
- protected boolean isForward;
- protected int[] lowKeyFields; // fields in input tuple to be used as low keys
- protected int[] highKeyFields; // fields in input tuple to be used as high
- // keys
+ // fields in input tuple to be used as low keys
+ protected int[] lowKeyFields;
+ // fields in input tuple to be used as high keys
+ protected int[] highKeyFields;
protected boolean lowKeyInclusive;
protected boolean highKeyInclusive;
- public BTreeSearchOperatorDescriptor(JobSpecification spec, RecordDescriptor recDesc,
+ public BTreeSearchOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc,
IStorageManagerInterface storageManager, IIndexRegistryProvider<IIndex> indexRegistryProvider,
- IFileSplitProvider fileSplitProvider, ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] comparatorFactories, boolean isForward, int[] lowKeyFields, int[] highKeyFields,
- boolean lowKeyInclusive, boolean highKeyInclusive, IIndexDataflowHelperFactory dataflowHelperFactory) {
- super(spec, 1, 1, recDesc, storageManager, indexRegistryProvider, fileSplitProvider, interiorFrameFactory,
- leafFrameFactory, typeTraits, comparatorFactories, dataflowHelperFactory);
- this.isForward = isForward;
+ IFileSplitProvider fileSplitProvider, ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] comparatorFactories, int[] lowKeyFields, int[] highKeyFields,
+ boolean lowKeyInclusive, boolean highKeyInclusive, IIndexDataflowHelperFactory dataflowHelperFactory,
+ boolean retainInput, IOperationCallbackProvider opCallbackProvider) {
+ super(spec, 1, 1, recDesc, storageManager, indexRegistryProvider, fileSplitProvider, typeTraits,
+ comparatorFactories, dataflowHelperFactory, null, retainInput, opCallbackProvider);
this.lowKeyFields = lowKeyFields;
this.highKeyFields = highKeyFields;
this.lowKeyInclusive = lowKeyInclusive;
@@ -57,9 +56,9 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider,
- int partition, int nPartitions) {
- return new BTreeSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, isForward, lowKeyFields,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ return new BTreeSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, lowKeyFields,
highKeyFields, lowKeyInclusive, highKeyInclusive);
}
-}
\ No newline at end of file
+}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
index 8a8c5ce..b284847 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
@@ -14,65 +14,30 @@
*/
package edu.uci.ics.hyracks.storage.am.btree.dataflow;
-import java.io.DataOutput;
-import java.nio.ByteBuffer;
-
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.PermutingFrameTupleReference;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexSearchOperatorNodePushable;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-public class BTreeSearchOperatorNodePushable extends AbstractUnaryInputUnaryOutputOperatorNodePushable {
- protected TreeIndexDataflowHelper treeIndexHelper;
- protected FrameTupleAccessor accessor;
-
- protected ByteBuffer writeBuffer;
- protected FrameTupleAppender appender;
- protected ArrayTupleBuilder tb;
- protected DataOutput dos;
-
- protected BTree btree;
- protected boolean isForward;
- protected PermutingFrameTupleReference lowKey;
- protected PermutingFrameTupleReference highKey;
- protected boolean lowKeyInclusive;
- protected boolean highKeyInclusive;
- protected RangePredicate rangePred;
- protected MultiComparator lowKeySearchCmp;
- protected MultiComparator highKeySearchCmp;
- protected ITreeIndexCursor cursor;
- protected ITreeIndexFrame cursorFrame;
- protected ITreeIndexAccessor indexAccessor;
-
- protected RecordDescriptor recDesc;
+public class BTreeSearchOperatorNodePushable extends TreeIndexSearchOperatorNodePushable {
+ protected PermutingFrameTupleReference lowKey;
+ protected PermutingFrameTupleReference highKey;
+ protected boolean lowKeyInclusive;
+ protected boolean highKeyInclusive;
+ protected MultiComparator lowKeySearchCmp;
+ protected MultiComparator highKeySearchCmp;
public BTreeSearchOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition, IRecordDescriptorProvider recordDescProvider, boolean isForward, int[] lowKeyFields,
- int[] highKeyFields, boolean lowKeyInclusive, boolean highKeyInclusive) {
- treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(
- opDesc, ctx, partition, false);
- this.isForward = isForward;
+ int partition, IRecordDescriptorProvider recordDescProvider, int[] lowKeyFields, int[] highKeyFields,
+ boolean lowKeyInclusive, boolean highKeyInclusive) {
+ super(opDesc, ctx, partition, recordDescProvider);
this.lowKeyInclusive = lowKeyInclusive;
this.highKeyInclusive = highKeyInclusive;
- this.recDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getOperatorId(), 0);
if (lowKeyFields != null && lowKeyFields.length > 0) {
lowKey = new PermutingFrameTupleReference();
lowKey.setFieldPermutation(lowKeyFields);
@@ -84,104 +49,19 @@
}
@Override
- public void open() throws HyracksDataException {
- AbstractTreeIndexOperatorDescriptor opDesc = (AbstractTreeIndexOperatorDescriptor) treeIndexHelper
- .getOperatorDescriptor();
- accessor = new FrameTupleAccessor(treeIndexHelper.getHyracksTaskContext().getFrameSize(), recDesc);
-
- cursorFrame = opDesc.getTreeIndexLeafFactory().createFrame();
- setCursor();
- writer.open();
-
- try {
- treeIndexHelper.init();
- btree = (BTree) treeIndexHelper.getIndex();
-
- // Construct range predicate.
- lowKeySearchCmp = BTreeUtils.getSearchMultiComparator(btree.getMultiComparator(), lowKey);
- highKeySearchCmp = BTreeUtils.getSearchMultiComparator(btree.getMultiComparator(), highKey);
- rangePred = new RangePredicate(isForward, null, null, lowKeyInclusive, highKeyInclusive, lowKeySearchCmp,
- highKeySearchCmp);
-
- writeBuffer = treeIndexHelper.getHyracksTaskContext().allocateFrame();
- tb = new ArrayTupleBuilder(btree.getFieldCount());
- dos = tb.getDataOutput();
- appender = new FrameTupleAppender(treeIndexHelper.getHyracksTaskContext().getFrameSize());
- appender.reset(writeBuffer, true);
- indexAccessor = btree.createAccessor();
- } catch (Exception e) {
- treeIndexHelper.deinit();
- throw new HyracksDataException(e);
+ protected void resetSearchPredicate(int tupleIndex) {
+ if (lowKey != null) {
+ lowKey.reset(accessor, tupleIndex);
}
- }
-
- protected void setCursor() {
- cursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) cursorFrame, false);
- }
-
- protected void writeSearchResults() throws Exception {
- while (cursor.hasNext()) {
- tb.reset();
- cursor.next();
-
- ITupleReference tuple = cursor.getTuple();
- for (int i = 0; i < tuple.getFieldCount(); i++) {
- dos.write(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
- tb.addFieldEndOffset();
- }
-
- if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
- FrameUtils.flushFrame(writeBuffer, writer);
- appender.reset(writeBuffer, true);
- if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
- throw new IllegalStateException();
- }
- }
+ if (highKey != null) {
+ highKey.reset(accessor, tupleIndex);
}
}
@Override
- public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
- accessor.reset(buffer);
- int tupleCount = accessor.getTupleCount();
- try {
- for (int i = 0; i < tupleCount; i++) {
- if (lowKey != null) {
- lowKey.reset(accessor, i);
- }
- if (highKey != null) {
- highKey.reset(accessor, i);
- }
- rangePred.setLowKey(lowKey, lowKeyInclusive);
- rangePred.setHighKey(highKey, highKeyInclusive);
- cursor.reset();
- indexAccessor.search(cursor, rangePred);
- writeSearchResults();
- }
- } catch (Exception e) {
- throw new HyracksDataException(e);
- }
- }
-
- @Override
- public void close() throws HyracksDataException {
- try {
- if (appender.getTupleCount() > 0) {
- FrameUtils.flushFrame(writeBuffer, writer);
- }
- writer.close();
- try {
- cursor.close();
- } catch (Exception e) {
- throw new HyracksDataException(e);
- }
- } finally {
- treeIndexHelper.deinit();
- }
- }
-
- @Override
- public void fail() throws HyracksDataException {
- writer.fail();
+ protected ISearchPredicate createSearchPredicate() {
+ lowKeySearchCmp = BTreeUtils.getSearchMultiComparator(treeIndex.getComparatorFactories(), lowKey);
+ highKeySearchCmp = BTreeUtils.getSearchMultiComparator(treeIndex.getComparatorFactories(), highKey);
+ return new RangePredicate(lowKey, highKey, lowKeyInclusive, highKeyInclusive, lowKeySearchCmp, highKeySearchCmp);
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorDescriptor.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorDescriptor.java
index 8cde863..1fc5cb8 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorDescriptor.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.btree.dataflow;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
@@ -6,9 +21,9 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.common.api.ITupleUpdaterFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
@@ -18,26 +33,25 @@
public class BTreeUpdateSearchOperatorDescriptor extends BTreeSearchOperatorDescriptor {
private static final long serialVersionUID = 1L;
-
+
private final ITupleUpdaterFactory tupleUpdaterFactory;
-
- public BTreeUpdateSearchOperatorDescriptor(JobSpecification spec, RecordDescriptor recDesc,
+
+ public BTreeUpdateSearchOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc,
IStorageManagerInterface storageManager, IIndexRegistryProvider<IIndex> indexRegistryProvider,
- IFileSplitProvider fileSplitProvider, ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] comparatorFactories, boolean isForward, int[] lowKeyFields, int[] highKeyFields,
+ IFileSplitProvider fileSplitProvider, ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] comparatorFactories, int[] lowKeyFields, int[] highKeyFields,
boolean lowKeyInclusive, boolean highKeyInclusive, IIndexDataflowHelperFactory dataflowHelperFactory,
- ITupleUpdaterFactory tupleUpdaterFactory) {
- super(spec, recDesc, storageManager, indexRegistryProvider, fileSplitProvider, interiorFrameFactory,
- leafFrameFactory, typeTraits, comparatorFactories, isForward, lowKeyFields, highKeyFields, lowKeyInclusive,
- highKeyInclusive, dataflowHelperFactory);
+ boolean retainInput, IOperationCallbackProvider opCallbackProvider, ITupleUpdaterFactory tupleUpdaterFactory) {
+ super(spec, recDesc, storageManager, indexRegistryProvider, fileSplitProvider, typeTraits, comparatorFactories,
+ lowKeyFields, highKeyFields, lowKeyInclusive, highKeyInclusive, dataflowHelperFactory,
+ retainInput, opCallbackProvider);
this.tupleUpdaterFactory = tupleUpdaterFactory;
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider,
- int partition, int nPartitions) {
- return new BTreeUpdateSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, isForward, lowKeyFields,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ return new BTreeUpdateSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, lowKeyFields,
highKeyFields, lowKeyInclusive, highKeyInclusive, tupleUpdaterFactory.createTupleUpdater());
}
}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorNodePushable.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorNodePushable.java
index 06a42bd..1141639 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorNodePushable.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.btree.dataflow;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
@@ -6,33 +21,38 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITupleUpdater;
import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
public class BTreeUpdateSearchOperatorNodePushable extends BTreeSearchOperatorNodePushable {
private final ITupleUpdater tupleUpdater;
-
- public BTreeUpdateSearchOperatorNodePushable(
- AbstractTreeIndexOperatorDescriptor opDesc,
- IHyracksTaskContext ctx, int partition,
- IRecordDescriptorProvider recordDescProvider, boolean isForward,
- int[] lowKeyFields, int[] highKeyFields, boolean lowKeyInclusive,
- boolean highKeyInclusive, ITupleUpdater tupleUpdater) {
- super(opDesc, ctx, partition, recordDescProvider, isForward, lowKeyFields,
- highKeyFields, lowKeyInclusive, highKeyInclusive);
- this.tupleUpdater = tupleUpdater;
- }
- @Override
- protected void setCursor() {
- cursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) cursorFrame, true);
+ public BTreeUpdateSearchOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
+ int partition, IRecordDescriptorProvider recordDescProvider, int[] lowKeyFields, int[] highKeyFields,
+ boolean lowKeyInclusive, boolean highKeyInclusive, ITupleUpdater tupleUpdater) {
+ super(opDesc, ctx, partition, recordDescProvider, lowKeyFields, highKeyFields, lowKeyInclusive,
+ highKeyInclusive);
+ this.tupleUpdater = tupleUpdater;
}
-
- @Override
- protected void writeSearchResults() throws Exception {
+
+ @Override
+ protected ITreeIndexCursor createCursor() {
+ return new BTreeRangeSearchCursor((IBTreeLeafFrame) cursorFrame, true);
+ }
+
+ @Override
+ protected void writeSearchResults(int tupleIndex) throws Exception {
while (cursor.hasNext()) {
tb.reset();
cursor.next();
+ if (retainInput) {
+ frameTuple.reset(accessor, tupleIndex);
+ for (int i = 0; i < frameTuple.getFieldCount(); i++) {
+ dos.write(frameTuple.getFieldData(i), frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
+ tb.addFieldEndOffset();
+ }
+ }
ITupleReference tuple = cursor.getTuple();
tupleUpdater.updateTuple(tuple);
for (int i = 0; i < tuple.getFieldCount(); i++) {
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
index 0aba70b..fb2e833 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
@@ -58,8 +58,7 @@
protected static final int uncompressedTupleCountOff = smFlagOff + 1; // 22
protected static final int prefixTupleCountOff = uncompressedTupleCountOff + 4; // 26
- protected static final int prevLeafOff = prefixTupleCountOff + 4; // 30
- protected static final int nextLeafOff = prevLeafOff + 4; // 34
+ protected static final int nextLeafOff = prefixTupleCountOff + 4; // 30
protected ICachedPage page = null;
protected ByteBuffer buf = null;
@@ -252,7 +251,7 @@
int freeSpace = buf.getInt(freeSpaceOff);
int bytesWritten = tupleWriter.writeTupleFields(tuple, numPrefixFields,
- tuple.getFieldCount() - numPrefixFields, buf, freeSpace);
+ tuple.getFieldCount() - numPrefixFields, buf.array(), freeSpace);
buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + bytesWritten);
@@ -315,11 +314,11 @@
if (inPlace) {
// Overwrite the old tuple suffix in place.
- bytesWritten = tupleWriter.writeTupleFields(newTuple, numPrefixFields, fieldCount - numPrefixFields, buf, suffixTupleStartOff);
+ bytesWritten = tupleWriter.writeTupleFields(newTuple, numPrefixFields, fieldCount - numPrefixFields, buf.array(), suffixTupleStartOff);
} else {
// Insert the new tuple suffix at the end of the free space, and change the slot value (effectively "deleting" the old tuple).
int newSuffixTupleStartOff = buf.getInt(freeSpaceOff);
- bytesWritten = tupleWriter.writeTupleFields(newTuple, numPrefixFields, fieldCount - numPrefixFields, buf, newSuffixTupleStartOff);
+ bytesWritten = tupleWriter.writeTupleFields(newTuple, numPrefixFields, fieldCount - numPrefixFields, buf.array(), newSuffixTupleStartOff);
// Update slot value using the same prefix slot num.
slotManager.setSlot(tupleSlotOff, slotManager.encodeSlotFields(prefixSlotNum, newSuffixTupleStartOff));
// Update contiguous free space pointer.
@@ -343,7 +342,6 @@
buf.putInt(prefixTupleCountOff, 0);
buf.put(levelOff, level);
buf.put(smFlagOff, (byte) 0);
- buf.putInt(prevLeafOff, -1);
buf.putInt(nextLeafOff, -1);
}
@@ -368,6 +366,35 @@
}
@Override
+ public int findUpsertTupleIndex(ITupleReference tuple) throws TreeIndexException {
+ int slot = slotManager.findSlot(tuple, frameTuple, framePrefixTuple, cmp, FindTupleMode.INCLUSIVE,
+ FindTupleNoExactMatchPolicy.HIGHER_KEY);
+ int tupleIndex = slotManager.decodeSecondSlotField(slot);
+ // Error indicator is set if there is an exact match.
+ if (tupleIndex == slotManager.getErrorIndicator()) {
+ throw new BTreeDuplicateKeyException("Trying to insert duplicate key into leaf node.");
+ }
+ return slot;
+ }
+
+ @Override
+ public ITupleReference getUpsertBeforeTuple(ITupleReference tuple, int targetTupleIndex) throws TreeIndexException {
+ int tupleIndex = slotManager.decodeSecondSlotField(targetTupleIndex);
+ // Examine the tuple index to determine whether it is valid or not.
+ if (tupleIndex != slotManager.getGreatestKeyIndicator()) {
+ // We need to check the key to determine whether it's an insert or an update.
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ if (cmp.compare(tuple, frameTuple) == 0) {
+ // The keys match, it's an update.
+ return frameTuple;
+ }
+ }
+ // Either the tuple index is a special indicator, or the keys don't match.
+ // In those cases, we are definitely dealing with an insert.
+ return null;
+ }
+
+ @Override
public int findUpdateTupleIndex(ITupleReference tuple) throws TreeIndexException {
int slot = slotManager.findSlot(tuple, frameTuple, framePrefixTuple, cmp, FindTupleMode.EXACT,
FindTupleNoExactMatchPolicy.HIGHER_KEY);
@@ -402,7 +429,6 @@
strBuilder.append("smFlagOff: " + smFlagOff + "\n");
strBuilder.append("uncompressedTupleCountOff: " + uncompressedTupleCountOff + "\n");
strBuilder.append("prefixTupleCountOff: " + prefixTupleCountOff + "\n");
- strBuilder.append("prevLeafOff: " + prevLeafOff + "\n");
strBuilder.append("nextLeafOff: " + nextLeafOff + "\n");
return strBuilder.toString();
}
@@ -493,7 +519,7 @@
}
int bytesWritten = tupleWriter.writeTupleFields(tuple, fieldsToTruncate, tuple.getFieldCount()
- - fieldsToTruncate, buf, freeSpace);
+ - fieldsToTruncate, buf.array(), freeSpace);
// insert slot
int prefixSlotNum = FieldPrefixSlotManager.TUPLE_UNCOMPRESSED;
@@ -634,7 +660,7 @@
int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0, cmp.getKeyFieldCount());
splitKey.initData(splitKeySize);
- tupleWriter.writeTupleFields(frameTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer(), 0);
+ tupleWriter.writeTupleFields(frameTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer().array(), 0);
splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer(), 0);
}
@@ -658,20 +684,10 @@
}
@Override
- public void setPrevLeaf(int page) {
- buf.putInt(prevLeafOff, page);
- }
-
- @Override
public int getNextLeaf() {
return buf.getInt(nextLeafOff);
}
- @Override
- public int getPrevLeaf() {
- return buf.getInt(prevLeafOff);
- }
-
public int getUncompressedTupleCount() {
return buf.getInt(uncompressedTupleCountOff);
}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
index 6173440..d2cb2c6 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
@@ -77,7 +77,7 @@
public void insert(ITupleReference tuple, int tupleIndex) {
int slotOff = slotManager.insertSlot(tupleIndex, buf.getInt(freeSpaceOff));
int freeSpace = buf.getInt(freeSpaceOff);
- int bytesWritten = tupleWriter.writeTupleFields(tuple, 0, tuple.getFieldCount(), buf, freeSpace);
+ int bytesWritten = tupleWriter.writeTupleFields(tuple, 0, tuple.getFieldCount(), buf.array(), freeSpace);
System.arraycopy(tuple.getFieldData(tuple.getFieldCount() - 1), getLeftChildPageOff(tuple), buf.array(),
freeSpace + bytesWritten, childPtrSize);
int tupleSize = bytesWritten + childPtrSize;
@@ -280,29 +280,15 @@
ITupleReference tuple = null;
FindTupleMode fsm = null;
// The target comparator may be on a prefix of the BTree key fields.
- MultiComparator targetCmp = null;
- if (pred.isForward()) {
- tuple = pred.getLowKey();
- if (tuple == null) {
- return getLeftmostChildPageId();
- }
- if (pred.isLowKeyInclusive()) {
- fsm = FindTupleMode.INCLUSIVE;
- } else {
- fsm = FindTupleMode.EXCLUSIVE;
- }
- targetCmp = pred.getLowKeyComparator();
+ MultiComparator targetCmp = pred.getLowKeyComparator();;
+ tuple = pred.getLowKey();
+ if (tuple == null) {
+ return getLeftmostChildPageId();
+ }
+ if (pred.isLowKeyInclusive()) {
+ fsm = FindTupleMode.INCLUSIVE;
} else {
- tuple = pred.getHighKey();
- if (tuple == null) {
- return getRightmostChildPageId();
- }
- if (pred.isHighKeyInclusive()) {
- fsm = FindTupleMode.EXCLUSIVE;
- } else {
- fsm = FindTupleMode.INCLUSIVE;
- }
- targetCmp = pred.getHighKeyComparator();
+ fsm = FindTupleMode.EXCLUSIVE;
}
// Search for a matching key.
int tupleIndex = slotManager.findTupleIndex(tuple, frameTuple, targetCmp, fsm,
@@ -320,37 +306,20 @@
int origTupleOff = slotManager.getTupleOff(slotOff);
cmpFrameTuple.resetByTupleOffset(buf, origTupleOff);
int cmpTupleOff = origTupleOff;
- if (pred.isForward()) {
- // The answer set begins with the lowest key matching the prefix.
- // We must follow the child pointer of the lowest (leftmost) key
- // matching the given prefix.
- int maxSlotOff = buf.capacity();
- slotOff += slotManager.getSlotSize();
- while (slotOff < maxSlotOff) {
- cmpTupleOff = slotManager.getTupleOff(slotOff);
- frameTuple.resetByTupleOffset(buf, cmpTupleOff);
- if (targetCmp.compare(cmpFrameTuple, frameTuple) != 0) {
- break;
- }
- slotOff += slotManager.getSlotSize();
- }
- slotOff -= slotManager.getSlotSize();
- } else {
- // The answer set begins with the highest key matching the prefix.
- // We must follow the child pointer of the highest (rightmost) key
- // matching the given prefix.
- int minSlotOff = slotManager.getSlotEndOff() - slotManager.getSlotSize();
- slotOff -= slotManager.getSlotSize();
- while (slotOff > minSlotOff) {
- cmpTupleOff = slotManager.getTupleOff(slotOff);
- frameTuple.resetByTupleOffset(buf, cmpTupleOff);
- if (targetCmp.compare(cmpFrameTuple, frameTuple) != 0) {
- break;
- }
- slotOff -= slotManager.getSlotSize();
+ // The answer set begins with the lowest key matching the prefix.
+ // We must follow the child pointer of the lowest (leftmost) key
+ // matching the given prefix.
+ int maxSlotOff = buf.capacity();
+ slotOff += slotManager.getSlotSize();
+ while (slotOff < maxSlotOff) {
+ cmpTupleOff = slotManager.getTupleOff(slotOff);
+ frameTuple.resetByTupleOffset(buf, cmpTupleOff);
+ if (targetCmp.compare(cmpFrameTuple, frameTuple) != 0) {
+ break;
}
slotOff += slotManager.getSlotSize();
}
+ slotOff -= slotManager.getSlotSize();
frameTuple.resetByTupleOffset(buf, slotManager.getTupleOff(slotOff));
int childPageOff = getLeftChildPageOff(frameTuple);
return buf.getInt(childPageOff);
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
index 4856595..4b7f44b 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
@@ -32,8 +32,7 @@
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFrame {
- protected static final int prevLeafOff = smFlagOff + 1;
- protected static final int nextLeafOff = prevLeafOff + 4;
+ protected static final int nextLeafOff = smFlagOff + 1;
private MultiComparator cmp;
public BTreeNSMLeafFrame(ITreeIndexTupleWriter tupleWriter) {
@@ -43,7 +42,6 @@
@Override
public void initBuffer(byte level) {
super.initBuffer(level);
- buf.putInt(prevLeafOff, -1);
buf.putInt(nextLeafOff, -1);
}
@@ -53,21 +51,11 @@
}
@Override
- public void setPrevLeaf(int page) {
- buf.putInt(prevLeafOff, page);
- }
-
- @Override
public int getNextLeaf() {
return buf.getInt(nextLeafOff);
}
@Override
- public int getPrevLeaf() {
- return buf.getInt(prevLeafOff);
- }
-
- @Override
public int findInsertTupleIndex(ITupleReference tuple) throws TreeIndexException {
int tupleIndex = slotManager.findTupleIndex(tuple, frameTuple, cmp, FindTupleMode.EXCLUSIVE_ERROR_IF_EXISTS,
FindTupleNoExactMatchPolicy.HIGHER_KEY);
@@ -83,18 +71,42 @@
int tupleIndex = slotManager.findTupleIndex(tuple, frameTuple, cmp, FindTupleMode.EXACT,
FindTupleNoExactMatchPolicy.HIGHER_KEY);
// Error indicator is set if there is no exact match.
- if (tupleIndex == slotManager.getErrorIndicator()) {
+ if (tupleIndex == slotManager.getErrorIndicator() || tupleIndex == slotManager.getGreatestKeyIndicator()) {
throw new BTreeNonExistentKeyException("Trying to update a tuple with a nonexistent key in leaf node.");
}
return tupleIndex;
}
@Override
+ public int findUpsertTupleIndex(ITupleReference tuple) throws TreeIndexException {
+ int tupleIndex = slotManager.findTupleIndex(tuple, frameTuple, cmp, FindTupleMode.INCLUSIVE,
+ FindTupleNoExactMatchPolicy.HIGHER_KEY);
+ // Just return the found tupleIndex. The caller will make the final decision whether to insert or update.
+ return tupleIndex;
+ }
+
+ @Override
+ public ITupleReference getUpsertBeforeTuple(ITupleReference tuple, int targetTupleIndex) throws TreeIndexException {
+ // Examine the tuple index to determine whether it is valid or not.
+ if (targetTupleIndex != slotManager.getGreatestKeyIndicator()) {
+ // We need to check the key to determine whether it's an insert or an update.
+ frameTuple.resetByTupleIndex(this, targetTupleIndex);
+ if (cmp.compare(tuple, frameTuple) == 0) {
+ // The keys match, it's an update.
+ return frameTuple;
+ }
+ }
+ // Either the tuple index is a special indicator, or the keys don't match.
+ // In those cases, we are definitely dealing with an insert.
+ return null;
+ }
+
+ @Override
public int findDeleteTupleIndex(ITupleReference tuple) throws TreeIndexException {
int tupleIndex = slotManager.findTupleIndex(tuple, frameTuple, cmp, FindTupleMode.EXACT,
FindTupleNoExactMatchPolicy.HIGHER_KEY);
// Error indicator is set if there is no exact match.
- if (tupleIndex == slotManager.getErrorIndicator()) {
+ if (tupleIndex == slotManager.getErrorIndicator() || tupleIndex == slotManager.getGreatestKeyIndicator()) {
throw new BTreeNonExistentKeyException("Trying to delete a tuple with a nonexistent key in leaf node.");
}
return tupleIndex;
@@ -103,7 +115,7 @@
@Override
public void insert(ITupleReference tuple, int tupleIndex) {
int freeSpace = buf.getInt(freeSpaceOff);
- slotManager.insertSlot(tupleIndex, freeSpace);
+ slotManager.insertSlot(tupleIndex, freeSpace);
int bytesWritten = tupleWriter.writeTuple(tuple, buf.array(), freeSpace);
buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + bytesWritten);
@@ -117,7 +129,7 @@
@Override
public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey) throws TreeIndexException {
- ByteBuffer right = rightFrame.getBuffer();
+ ByteBuffer right = rightFrame.getBuffer();
int tupleCount = getTupleCount();
// Find split point, and determine into which frame the new tuple should be inserted into.
@@ -162,7 +174,7 @@
frameTuple.resetByTupleOffset(buf, tupleOff);
int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0, cmp.getKeyFieldCount());
splitKey.initData(splitKeySize);
- tupleWriter.writeTupleFields(frameTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer(), 0);
+ tupleWriter.writeTupleFields(frameTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer().array(), 0);
splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer(), 0);
}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
index 4113e9f..89e8870 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
@@ -19,6 +19,7 @@
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
@@ -26,10 +27,13 @@
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeNonExistentKeyException;
import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeNotUpdateableException;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrame;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
@@ -39,7 +43,6 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
-import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
@@ -60,24 +63,24 @@
private final IFreePageManager freePageManager;
private final IBufferCache bufferCache;
+ private final IOperationCallback opCallback;
private final ITreeIndexFrameFactory interiorFrameFactory;
private final ITreeIndexFrameFactory leafFrameFactory;
private final int fieldCount;
- private final MultiComparator cmp;
+ private final IBinaryComparatorFactory[] cmpFactories;
private final ReadWriteLock treeLatch;
- private final RangePredicate diskOrderScanPredicate;
private int fileId;
- public BTree(IBufferCache bufferCache, int fieldCount, MultiComparator cmp, IFreePageManager freePageManager,
+ public BTree(IBufferCache bufferCache, IOperationCallback opCallback, int fieldCount, IBinaryComparatorFactory[] cmpFactories, IFreePageManager freePageManager,
ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
this.bufferCache = bufferCache;
+ this.opCallback = opCallback;
this.fieldCount = fieldCount;
- this.cmp = cmp;
+ this.cmpFactories = cmpFactories;
this.interiorFrameFactory = interiorFrameFactory;
this.leafFrameFactory = leafFrameFactory;
this.freePageManager = freePageManager;
this.treeLatch = new ReentrantReadWriteLock(true);
- this.diskOrderScanPredicate = new RangePredicate(true, null, null, true, true, cmp, cmp);
}
@Override
@@ -87,6 +90,7 @@
ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
this.fileId = fileId;
+ freePageManager.open(fileId);
freePageManager.init(metaFrame, rootPage);
initRoot(leafFrame, true);
} finally {
@@ -95,32 +99,23 @@
}
@Override
- public void open(int fileId) {
+ public void open(int fileId) {
this.fileId = fileId;
+ freePageManager.open(fileId);
}
@Override
public void close() {
fileId = -1;
+ freePageManager.close();
}
- private void addFreePages(BTreeOpContext ctx) throws HyracksDataException {
- for (int i = 0; i < ctx.freePages.size(); i++) {
- // Root page is special, never add it to free pages.
- if (ctx.freePages.get(i) != rootPage) {
- freePageManager.addFreePage(ctx.metaFrame, ctx.freePages.get(i));
- }
- }
- ctx.freePages.clear();
- }
-
private void diskOrderScan(ITreeIndexCursor icursor, BTreeOpContext ctx) throws HyracksDataException {
TreeDiskOrderScanCursor cursor = (TreeDiskOrderScanCursor) icursor;
ctx.reset();
-
+ RangePredicate diskOrderScanPred = new RangePredicate(null, null, true, true, ctx.cmp, ctx.cmp);
int currentPageId = rootPage;
int maxPageId = freePageManager.getMaxPage(ctx.metaFrame);
-
ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false);
page.acquireReadLatch();
try {
@@ -129,7 +124,7 @@
cursor.setCurrentPageId(currentPageId);
cursor.setMaxPageId(maxPageId);
ctx.cursorInitialState.setPage(page);
- cursor.open(ctx.cursorInitialState, diskOrderScanPredicate);
+ cursor.open(ctx.cursorInitialState, diskOrderScanPred);
} catch (Exception e) {
page.releaseReadLatch();
bufferCache.unpin(page);
@@ -138,22 +133,22 @@
}
private void search(ITreeIndexCursor cursor, ISearchPredicate searchPred, BTreeOpContext ctx)
- throws TreeIndexException, HyracksDataException, PageAllocationException {
+ throws TreeIndexException, HyracksDataException {
ctx.reset();
ctx.pred = (RangePredicate) searchPred;
ctx.cursor = cursor;
// simple index scan
if (ctx.pred.getLowKeyComparator() == null) {
- ctx.pred.setLowKeyComparator(cmp);
+ ctx.pred.setLowKeyComparator(ctx.cmp);
}
if (ctx.pred.getHighKeyComparator() == null) {
- ctx.pred.setHighKeyComparator(cmp);
+ ctx.pred.setHighKeyComparator(ctx.cmp);
}
// we use this loop to deal with possibly multiple operation restarts
// due to ongoing structure modifications during the descent
boolean repeatOp = true;
while (repeatOp && ctx.opRestarts < MAX_RESTARTS) {
- performOp(rootPage, null, ctx);
+ performOp(rootPage, null, true, ctx);
// if we reach this stage then we need to restart from the (possibly
// new) root
if (!ctx.pageLsns.isEmpty() && ctx.pageLsns.getLast() == RESTART_OP) {
@@ -199,45 +194,32 @@
}
}
- private void createNewRoot(BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void createNewRoot(BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
// Make sure the root is always in the same page.
ICachedPage leftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, ctx.splitKey.getLeftPage()),
false);
leftNode.acquireWriteLatch();
try {
- ICachedPage rightNode = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, ctx.splitKey.getRightPage()), false);
- rightNode.acquireWriteLatch();
+ int newLeftId = freePageManager.getFreePage(ctx.metaFrame);
+ ICachedPage newLeftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newLeftId), true);
+ newLeftNode.acquireWriteLatch();
try {
- int newLeftId = freePageManager.getFreePage(ctx.metaFrame);
- ICachedPage newLeftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newLeftId), true);
- newLeftNode.acquireWriteLatch();
- try {
- // Copy left child to new left child.
- System.arraycopy(leftNode.getBuffer().array(), 0, newLeftNode.getBuffer().array(), 0, newLeftNode
- .getBuffer().capacity());
- ctx.interiorFrame.setPage(newLeftNode);
- ctx.interiorFrame.setSmFlag(false);
- // Change sibling pointer if children are leaves.
- ctx.leafFrame.setPage(rightNode);
- if (ctx.leafFrame.isLeaf()) {
- ctx.leafFrame.setPrevLeaf(newLeftId);
- }
- // Initialize new root (leftNode becomes new root).
- ctx.interiorFrame.setPage(leftNode);
- ctx.interiorFrame.initBuffer((byte) (ctx.leafFrame.getLevel() + 1));
- // Will be cleared later in unsetSmPages.
- ctx.interiorFrame.setSmFlag(true);
- ctx.splitKey.setLeftPage(newLeftId);
- int targetTupleIndex = ctx.interiorFrame.findInsertTupleIndex(ctx.splitKey.getTuple());
- ctx.interiorFrame.insert(ctx.splitKey.getTuple(), targetTupleIndex);
- } finally {
- newLeftNode.releaseWriteLatch();
- bufferCache.unpin(newLeftNode);
- }
+ // Copy left child to new left child.
+ System.arraycopy(leftNode.getBuffer().array(), 0, newLeftNode.getBuffer().array(), 0, newLeftNode
+ .getBuffer().capacity());
+ ctx.interiorFrame.setPage(newLeftNode);
+ ctx.interiorFrame.setSmFlag(false);
+ // Initialize new root (leftNode becomes new root).
+ ctx.interiorFrame.setPage(leftNode);
+ ctx.interiorFrame.initBuffer((byte) (ctx.interiorFrame.getLevel() + 1));
+ // Will be cleared later in unsetSmPages.
+ ctx.interiorFrame.setSmFlag(true);
+ ctx.splitKey.setLeftPage(newLeftId);
+ int targetTupleIndex = ctx.interiorFrame.findInsertTupleIndex(ctx.splitKey.getTuple());
+ ctx.interiorFrame.insert(ctx.splitKey.getTuple(), targetTupleIndex);
} finally {
- rightNode.releaseWriteLatch();
- bufferCache.unpin(rightNode);
+ newLeftNode.releaseWriteLatch();
+ bufferCache.unpin(newLeftNode);
}
} finally {
leftNode.releaseWriteLatch();
@@ -245,19 +227,19 @@
}
}
- private void insertUpdateOrDelete(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void insertUpdateOrDelete(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
ctx.reset();
- ctx.pred.setLowKeyComparator(cmp);
- ctx.pred.setHighKeyComparator(cmp);
+ ctx.pred.setLowKeyComparator(ctx.cmp);
+ ctx.pred.setHighKeyComparator(ctx.cmp);
ctx.pred.setLowKey(tuple, true);
ctx.pred.setHighKey(tuple, true);
ctx.splitKey.reset();
- ctx.splitKey.getTuple().setFieldCount(cmp.getKeyFieldCount());
+ ctx.splitKey.getTuple().setFieldCount(ctx.cmp.getKeyFieldCount());
// We use this loop to deal with possibly multiple operation restarts
// due to ongoing structure modifications during the descent.
boolean repeatOp = true;
while (repeatOp && ctx.opRestarts < MAX_RESTARTS) {
- performOp(rootPage, null, ctx);
+ performOp(rootPage, null, true, ctx);
// Do we need to restart from the (possibly new) root?
if (!ctx.pageLsns.isEmpty() && ctx.pageLsns.getLast() == RESTART_OP) {
ctx.pageLsns.removeLast(); // pop the restart op indicator
@@ -265,44 +247,38 @@
}
// Split key propagated?
if (ctx.splitKey.getBuffer() != null) {
- if (ctx.op == IndexOp.DELETE) {
- // Reset level of root to zero.
- initRoot(ctx.leafFrame, false);
- } else {
- // Insert or update op. Create a new root.
- createNewRoot(ctx);
- }
+ // Insert or update op. Create a new root.
+ createNewRoot(ctx);
}
unsetSmPages(ctx);
- if (ctx.op == IndexOp.DELETE) {
- addFreePages(ctx);
- }
repeatOp = false;
}
}
- private void insert(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void insert(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
+ insertUpdateOrDelete(tuple, ctx);
+ }
+
+ private void upsert(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
insertUpdateOrDelete(tuple, ctx);
}
- private void update(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void update(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
// This call only allows updating of non-key fields.
// Updating a tuple's key necessitates deleting the old entry, and inserting the new entry.
// The user of the BTree is responsible for dealing with non-key updates (i.e., doing a delete + insert).
- if (fieldCount == cmp.getKeyFieldCount()) {
+ if (fieldCount == ctx.cmp.getKeyFieldCount()) {
throw new BTreeNotUpdateableException("Cannot perform updates when the entire tuple forms the key.");
}
insertUpdateOrDelete(tuple, ctx);
}
- private void delete(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void delete(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
insertUpdateOrDelete(tuple, ctx);
}
- private boolean insertLeaf(ICachedPage node, int pageId, ITupleReference tuple, BTreeOpContext ctx) throws Exception {
- ctx.leafFrame.setPage(node);
+ private boolean insertLeaf(ITupleReference tuple, int targetTupleIndex, int pageId, BTreeOpContext ctx) throws Exception {
boolean restartOp = false;
- int targetTupleIndex = ctx.leafFrame.findInsertTupleIndex(tuple);
FrameOpSpaceStatus spaceStatus = ctx.leafFrame.hasSpaceInsert(tuple);
switch (spaceStatus) {
case SUFFICIENT_CONTIGUOUS_SPACE: {
@@ -335,81 +311,57 @@
}
break;
}
- }
- node.releaseWriteLatch();
- bufferCache.unpin(node);
+ }
return restartOp;
}
private boolean performLeafSplit(int pageId, ITupleReference tuple, BTreeOpContext ctx) throws Exception {
- // Lock is released in unsetSmPages(), after sm has fully completed.
+ // We must never hold a latch on a page while waiting to obtain the tree
+ // latch, because it this could lead to a latch-deadlock.
+ // If we can't get the tree latch, we return, release our page latches,
+ // and restart the operation from one level above.
+ // Lock is released in unsetSmPages(), after sm has fully completed.
if (!treeLatch.writeLock().tryLock()) {
- return true;
+ return true;
}
- int rightSiblingPageId = ctx.leafFrame.getNextLeaf();
- ICachedPage rightSibling = null;
- if (rightSiblingPageId > 0) {
- rightSibling = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightSiblingPageId),
- false);
- }
+ int rightPageId = freePageManager.getFreePage(ctx.metaFrame);
+ ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId),
+ true);
+ rightNode.acquireWriteLatch();
try {
- int rightPageId = freePageManager.getFreePage(ctx.metaFrame);
- ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId),
- true);
- rightNode.acquireWriteLatch();
- try {
- IBTreeLeafFrame rightFrame = ctx.createLeafFrame();
- rightFrame.setPage(rightNode);
- rightFrame.initBuffer((byte) 0);
- rightFrame.setMultiComparator(cmp);
- ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey);
+ IBTreeLeafFrame rightFrame = ctx.createLeafFrame();
+ rightFrame.setPage(rightNode);
+ rightFrame.initBuffer((byte) 0);
+ rightFrame.setMultiComparator(ctx.cmp);
+ ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey);
- ctx.smPages.add(pageId);
- ctx.smPages.add(rightPageId);
- ctx.leafFrame.setSmFlag(true);
- rightFrame.setSmFlag(true);
+ ctx.smPages.add(pageId);
+ ctx.smPages.add(rightPageId);
+ ctx.leafFrame.setSmFlag(true);
+ rightFrame.setSmFlag(true);
- rightFrame.setNextLeaf(ctx.leafFrame.getNextLeaf());
- rightFrame.setPrevLeaf(pageId);
- ctx.leafFrame.setNextLeaf(rightPageId);
+ rightFrame.setNextLeaf(ctx.leafFrame.getNextLeaf());
+ ctx.leafFrame.setNextLeaf(rightPageId);
- // TODO: we just use increasing numbers as pageLsn,
- // we
- // should tie this together with the LogManager and
- // TransactionManager
- rightFrame.setPageLsn(rightFrame.getPageLsn() + 1);
- ctx.leafFrame.setPageLsn(ctx.leafFrame.getPageLsn() + 1);
+ // TODO: we just use increasing numbers as pageLsn,
+ // we
+ // should tie this together with the LogManager and
+ // TransactionManager
+ rightFrame.setPageLsn(rightFrame.getPageLsn() + 1);
+ ctx.leafFrame.setPageLsn(ctx.leafFrame.getPageLsn() + 1);
- ctx.splitKey.setPages(pageId, rightPageId);
-
- if (rightSibling != null) {
- rightSibling.acquireWriteLatch();
- try {
- // Reuse rightFrame for modification.
- rightFrame.setPage(rightSibling);
- rightFrame.setPrevLeaf(rightPageId);
- } finally {
- rightSibling.releaseWriteLatch();
- }
- }
- } finally {
- rightNode.releaseWriteLatch();
- bufferCache.unpin(rightNode);
- }
+ ctx.splitKey.setPages(pageId, rightPageId);
} catch (Exception e) {
treeLatch.writeLock().unlock();
throw e;
} finally {
- if (rightSibling != null) {
- bufferCache.unpin(rightSibling);
- }
+ rightNode.releaseWriteLatch();
+ bufferCache.unpin(rightNode);
}
return false;
}
- private boolean updateLeaf(ICachedPage node, int pageId, ITupleReference tuple, BTreeOpContext ctx) throws Exception {
- ctx.leafFrame.setPage(node);
- int oldTupleIndex = ctx.leafFrame.findUpdateTupleIndex(tuple);
+ private boolean updateLeaf(ITupleReference tuple, int oldTupleIndex, int pageId, BTreeOpContext ctx) throws Exception {
FrameOpSpaceStatus spaceStatus = ctx.leafFrame.hasSpaceUpdate(tuple, oldTupleIndex);
boolean restartOp = false;
switch (spaceStatus) {
@@ -448,11 +400,23 @@
break;
}
}
- node.releaseWriteLatch();
- bufferCache.unpin(node);
return restartOp;
}
+ private boolean upsertLeaf(ITupleReference tuple, int targetTupleIndex, int pageId, BTreeOpContext ctx) throws Exception {
+ boolean restartOp = false;
+ ITupleReference beforeTuple = ctx.leafFrame.getUpsertBeforeTuple(tuple, targetTupleIndex);
+ if (beforeTuple == null) {
+ opCallback.pre(null);
+ restartOp = insertLeaf(tuple, targetTupleIndex, pageId, ctx);
+ } else {
+ opCallback.pre(beforeTuple);
+ restartOp = updateLeaf(tuple, targetTupleIndex, pageId, ctx);
+ }
+ opCallback.post(tuple);
+ return restartOp;
+ }
+
private void insertInterior(ICachedPage node, int pageId, ITupleReference tuple, BTreeOpContext ctx)
throws Exception {
ctx.interiorFrame.setPage(node);
@@ -467,7 +431,7 @@
IBTreeFrame rightFrame = ctx.createInteriorFrame();
rightFrame.setPage(rightNode);
rightFrame.initBuffer((byte) ctx.interiorFrame.getLevel());
- rightFrame.setMultiComparator(cmp);
+ rightFrame.setMultiComparator(ctx.cmp);
// instead of creating a new split key, use the existing
// splitKey
ctx.interiorFrame.split(rightFrame, ctx.splitKey.getTuple(), ctx.splitKey);
@@ -508,147 +472,24 @@
}
private boolean deleteLeaf(ICachedPage node, int pageId, ITupleReference tuple, BTreeOpContext ctx) throws Exception {
- ctx.leafFrame.setPage(node);
+ // Simply delete the tuple, and don't do any rebalancing.
+ // This means that there could be underflow, even an empty page that is
+ // pointed to by an interior node.
+ if (ctx.leafFrame.getTupleCount() == 0) {
+ throw new BTreeNonExistentKeyException("Trying to delete a tuple with a nonexistent key in leaf node.");
+ }
int tupleIndex = ctx.leafFrame.findDeleteTupleIndex(tuple);
-
- // Will this leaf become empty?
- if (ctx.leafFrame.getTupleCount() > 1) {
- // Leaf will not become empty.
- ctx.leafFrame.delete(tuple, tupleIndex);
- node.releaseWriteLatch();
- bufferCache.unpin(node);
- return false;
- }
-
- // Leaf will become empty.
- IBTreeLeafFrame siblingFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
- siblingFrame.setMultiComparator(cmp);
- ICachedPage leftNode = null;
- ICachedPage rightNode = null;
- int nextLeaf = ctx.leafFrame.getNextLeaf();
- int prevLeaf = ctx.leafFrame.getPrevLeaf();
- // Try to get the tree latch, if it's already taken, then restart this operation
- // to avoid latch deadlock.
- if (!treeLatch.writeLock().tryLock()) {
- return true;
- }
- try {
- if (prevLeaf > 0) {
- leftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, prevLeaf), false);
- }
- try {
- if (nextLeaf > 0) {
- rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeaf), false);
- }
- try {
- try {
- ctx.leafFrame.delete(tuple, tupleIndex);
- // To propagate the deletion we only need to make the
- // splitKey != null.
- // Reuse data to identify which key to delete in the parent.
- ctx.splitKey.initData(1);
- } catch (Exception e) {
- // Don't propagate deletion.
- ctx.splitKey.reset();
- throw e;
- }
-
- // TODO: Tie together with logging.
- ctx.leafFrame.setPageLsn(ctx.leafFrame.getPageLsn() + 1);
- ctx.leafFrame.setLevel(freePageManager.getFreePageLevelIndicator());
-
- ctx.smPages.add(pageId);
- ctx.leafFrame.setSmFlag(true);
-
- node.releaseWriteLatch();
- bufferCache.unpin(node);
-
- if (leftNode != null) {
- leftNode.acquireWriteLatch();
- try {
- siblingFrame.setPage(leftNode);
- siblingFrame.setNextLeaf(nextLeaf);
- // TODO: Tie together with logging.
- siblingFrame.setPageLsn(siblingFrame.getPageLsn() + 1);
- } finally {
- leftNode.releaseWriteLatch();
- }
- }
-
- if (rightNode != null) {
- rightNode.acquireWriteLatch();
- try {
- siblingFrame.setPage(rightNode);
- siblingFrame.setPrevLeaf(prevLeaf);
- // TODO: Tie together with logging.
- siblingFrame.setPageLsn(siblingFrame.getPageLsn() + 1);
- } finally {
- rightNode.releaseWriteLatch();
- }
- }
- // Register pageId as a free.
- ctx.freePages.add(pageId);
- } finally {
- if (rightNode != null) {
- bufferCache.unpin(rightNode);
- }
- }
- } finally {
- if (leftNode != null) {
- bufferCache.unpin(leftNode);
- }
- }
- } catch (Exception e) {
- treeLatch.writeLock().unlock();
- throw e;
- }
+ ctx.leafFrame.delete(tuple, tupleIndex);
return false;
}
- private void deleteInterior(ICachedPage node, int pageId, ITupleReference tuple, BTreeOpContext ctx)
- throws Exception {
- ctx.interiorFrame.setPage(node);
-
- int tupleIndex = ctx.interiorFrame.findDeleteTupleIndex(tuple);
-
- // this means there is only a child pointer but no key, this case
- // propagates the split
- if (ctx.interiorFrame.getTupleCount() == 0) {
- ctx.interiorFrame.setPageLsn(ctx.interiorFrame.getPageLsn() + 1); // TODO:
- // tie
- // together
- // with
- // logging
- ctx.leafFrame.setLevel(freePageManager.getFreePageLevelIndicator());
- ctx.smPages.add(pageId);
- ctx.interiorFrame.setSmFlag(true);
- ctx.interiorFrame.setRightmostChildPageId(-1); // this node is
- // completely empty
- // register this pageId as a free page
- ctx.freePages.add(pageId);
-
- } else {
- ctx.interiorFrame.delete(tuple, tupleIndex);
- // TODO: Tie together with logging.
- ctx.interiorFrame.setPageLsn(ctx.interiorFrame.getPageLsn() + 1);
- // Don't propagate deletion.
- ctx.splitKey.reset();
- }
- }
-
- private final void acquireLatch(ICachedPage node, BTreeOpContext ctx, boolean isLeaf) {
+ private final boolean acquireLatch(ICachedPage node, BTreeOpContext ctx, boolean isLeaf) {
if (!isLeaf || (ctx.op == IndexOp.SEARCH && !ctx.cursor.exclusiveLatchNodes())) {
node.acquireReadLatch();
+ return true;
} else {
node.acquireWriteLatch();
- }
- }
-
- private final void releaseLatch(ICachedPage node, BTreeOpContext ctx, boolean isLeaf) {
- if (!isLeaf || (ctx.op == IndexOp.SEARCH && !ctx.cursor.exclusiveLatchNodes())) {
- node.releaseReadLatch();
- } else {
- node.releaseWriteLatch();
+ return false;
}
}
@@ -666,14 +507,14 @@
return isConsistent;
}
- private void performOp(int pageId, ICachedPage parent, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void performOp(int pageId, ICachedPage parent, boolean parentIsReadLatched, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
ctx.interiorFrame.setPage(node);
// this check performs an unprotected read in the page
// the following could happen: TODO fill out
boolean unsafeIsLeaf = ctx.interiorFrame.isLeaf();
- acquireLatch(node, ctx, unsafeIsLeaf);
+ boolean isReadLatched = acquireLatch(node, ctx, unsafeIsLeaf);
boolean smFlag = ctx.interiorFrame.getSmFlag();
// re-check leafness after latching
boolean isLeaf = ctx.interiorFrame.isLeaf();
@@ -682,10 +523,13 @@
// structure modification
ctx.pageLsns.add(ctx.interiorFrame.getPageLsn());
try {
- // latch coupling, note: parent should never be write latched,
- // otherwise something is wrong.
+ // Latch coupling: unlatch parent.
if (parent != null) {
- parent.releaseReadLatch();
+ if (parentIsReadLatched) {
+ parent.releaseReadLatch();
+ } else {
+ parent.releaseWriteLatch();
+ }
bufferCache.unpin(parent);
}
if (!isLeaf || smFlag) {
@@ -696,15 +540,18 @@
boolean repeatOp = true;
while (repeatOp && ctx.opRestarts < MAX_RESTARTS) {
int childPageId = ctx.interiorFrame.getChildPageId(ctx.pred);
- performOp(childPageId, node, ctx);
+ performOp(childPageId, node, isReadLatched, ctx);
if (!ctx.pageLsns.isEmpty() && ctx.pageLsns.getLast() == RESTART_OP) {
// Pop the restart op indicator.
- ctx.pageLsns.removeLast();
+ ctx.pageLsns.removeLast();
if (isConsistent(pageId, ctx)) {
- // Don't unpin and unlatch node again in recursive call.
- node = null;
- // Descend the tree again.
+ // Pin and latch page again, since it was unpinned and unlatched in call to performOp (passed as parent).
+ node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ node.acquireReadLatch();
+ ctx.interiorFrame.setPage(node);
+ isReadLatched = true;
+ // Descend the tree again.
continue;
} else {
// Pop pageLsn of this page (version seen by this op during descent).
@@ -717,28 +564,32 @@
switch (ctx.op) {
case INSERT:
- case UPDATE:
- case DELETE: {
+ case UPSERT:
+ case UPDATE: {
// Is there a propagated split key?
if (ctx.splitKey.getBuffer() != null) {
- node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- node.acquireWriteLatch();
+ ICachedPage interiorNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ interiorNode.acquireWriteLatch();
try {
- if (ctx.op == IndexOp.DELETE) {
- deleteInterior(node, pageId, ctx.pred.getLowKey(), ctx);
- } else {
- // Insert or update op. Both can cause split keys to propagate upwards.
- insertInterior(node, pageId, ctx.splitKey.getTuple(), ctx);
- }
+ // Insert or update op. Both can cause split keys to propagate upwards.
+ insertInterior(interiorNode, pageId, ctx.splitKey.getTuple(), ctx);
} finally {
- node.releaseWriteLatch();
- bufferCache.unpin(node);
+ interiorNode.releaseWriteLatch();
+ bufferCache.unpin(interiorNode);
}
} else {
unsetSmPages(ctx);
}
break;
}
+
+ case DELETE: {
+ if (ctx.splitKey.getBuffer() != null) {
+ throw new BTreeException("Split key was propagated during delete. Delete allows empty leaf pages.");
+ }
+ break;
+ }
+
default: {
// Do nothing for Search and DiskOrderScan.
break;
@@ -749,7 +600,11 @@
} // end while
} else { // smFlag
ctx.opRestarts++;
- releaseLatch(node, ctx, unsafeIsLeaf);
+ if (isReadLatched) {
+ node.releaseReadLatch();
+ } else {
+ node.releaseWriteLatch();
+ }
bufferCache.unpin(node);
// TODO: this should be an instant duration lock, how to do
@@ -770,13 +625,21 @@
} else { // isLeaf and !smFlag
// We may have to restart an op to avoid latch deadlock.
boolean restartOp = false;
+ ctx.leafFrame.setPage(node);
switch (ctx.op) {
- case INSERT: {
- restartOp = insertLeaf(node, pageId, ctx.pred.getLowKey(), ctx);
+ case INSERT: {
+ int targetTupleIndex = ctx.leafFrame.findInsertTupleIndex(ctx.pred.getLowKey());
+ restartOp = insertLeaf(ctx.pred.getLowKey(), targetTupleIndex, pageId, ctx);
+ break;
+ }
+ case UPSERT: {
+ int targetTupleIndex = ctx.leafFrame.findUpsertTupleIndex(ctx.pred.getLowKey());
+ restartOp = upsertLeaf(ctx.pred.getLowKey(), targetTupleIndex, pageId, ctx);
break;
}
case UPDATE: {
- restartOp = updateLeaf(node, pageId, ctx.pred.getLowKey(), ctx);
+ int oldTupleIndex = ctx.leafFrame.findUpdateTupleIndex(ctx.pred.getLowKey());
+ restartOp = updateLeaf(ctx.pred.getLowKey(), oldTupleIndex, pageId, ctx);
break;
}
case DELETE: {
@@ -789,6 +652,10 @@
break;
}
}
+ if (ctx.op != IndexOp.SEARCH) {
+ node.releaseWriteLatch();
+ bufferCache.unpin(node);
+ }
if (restartOp) {
ctx.pageLsns.removeLast();
ctx.pageLsns.add(RESTART_OP);
@@ -796,28 +663,35 @@
}
} catch (TreeIndexException e) {
if (!ctx.exceptionHandled) {
- releaseLatch(node, ctx, unsafeIsLeaf);
- bufferCache.unpin(node);
- ctx.exceptionHandled = true;
- }
- throw e;
- } catch (PageAllocationException e) {
- if (!ctx.exceptionHandled) {
- releaseLatch(node, ctx, unsafeIsLeaf);
- bufferCache.unpin(node);
- ctx.exceptionHandled = true;
+ if (node != null) {
+ if (isReadLatched) {
+ node.releaseReadLatch();
+ } else {
+ node.releaseWriteLatch();
+ }
+ bufferCache.unpin(node);
+ ctx.exceptionHandled = true;
+ }
}
throw e;
} catch (Exception e) {
e.printStackTrace();
- releaseLatch(node, ctx, unsafeIsLeaf);
- bufferCache.unpin(node);
+ if (node != null) {
+ if (isReadLatched) {
+ node.releaseReadLatch();
+ } else {
+ node.releaseWriteLatch();
+ }
+ bufferCache.unpin(node);
+ }
BTreeException wrappedException = new BTreeException(e);
+ ctx.exceptionHandled = true;
throw wrappedException;
}
}
- public final class BulkLoadContext implements IIndexBulkLoadContext {
+ public class BulkLoadContext implements IIndexBulkLoadContext {
+ public final MultiComparator cmp;
public final int slotSize;
public final int leafMaxBytes;
public final int interiorMaxBytes;
@@ -827,12 +701,12 @@
private final IBTreeLeafFrame leafFrame;
private final IBTreeInteriorFrame interiorFrame;
private final ITreeIndexMetaDataFrame metaFrame;
-
- private final ITreeIndexTupleWriter tupleWriter;
-
+ private final ITreeIndexTupleWriter tupleWriter;
+
public BulkLoadContext(float fillFactor, IBTreeLeafFrame leafFrame, IBTreeInteriorFrame interiorFrame,
- ITreeIndexMetaDataFrame metaFrame, MultiComparator cmp) throws HyracksDataException, PageAllocationException {
-
+ ITreeIndexMetaDataFrame metaFrame, IBinaryComparatorFactory[] cmpFactories) throws HyracksDataException {
+ this.cmp = MultiComparator.create(cmpFactories);
+
leafFrame.setMultiComparator(cmp);
interiorFrame.setMultiComparator(cmp);
@@ -862,7 +736,7 @@
nodeFrontiers.add(leafFrontier);
}
- private void addLevel() throws HyracksDataException, PageAllocationException {
+ private void addLevel() throws HyracksDataException {
NodeFrontier frontier = new NodeFrontier(tupleWriter.createTupleReference());
frontier.pageId = freePageManager.getFreePage(metaFrame);
frontier.page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, frontier.pageId), true);
@@ -874,7 +748,7 @@
}
}
- private void propagateBulk(BulkLoadContext ctx, int level) throws HyracksDataException, PageAllocationException {
+ private void propagateBulk(BulkLoadContext ctx, int level) throws HyracksDataException {
if (ctx.splitKey.getBuffer() == null)
return;
@@ -886,7 +760,7 @@
ctx.interiorFrame.setPage(frontier.page);
ITupleReference tuple = ctx.splitKey.getTuple();
- int spaceNeeded = ctx.tupleWriter.bytesRequired(tuple, 0, cmp.getKeyFieldCount()) + ctx.slotSize + 4;
+ int spaceNeeded = ctx.tupleWriter.bytesRequired(tuple, 0, ctx.cmp.getKeyFieldCount()) + ctx.slotSize + 4;
int spaceUsed = ctx.interiorFrame.getBuffer().capacity() - ctx.interiorFrame.getTotalFreeSpace();
if (spaceUsed + spaceNeeded > ctx.interiorMaxBytes) {
@@ -894,10 +768,10 @@
tuple = copyKey.getTuple();
frontier.lastTuple.resetByTupleIndex(ctx.interiorFrame, ctx.interiorFrame.getTupleCount() - 1);
- int splitKeySize = ctx.tupleWriter.bytesRequired(frontier.lastTuple, 0, cmp.getKeyFieldCount());
+ int splitKeySize = ctx.tupleWriter.bytesRequired(frontier.lastTuple, 0, ctx.cmp.getKeyFieldCount());
ctx.splitKey.initData(splitKeySize);
ctx.tupleWriter
- .writeTupleFields(frontier.lastTuple, 0, cmp.getKeyFieldCount(), ctx.splitKey.getBuffer(), 0);
+ .writeTupleFields(frontier.lastTuple, 0, ctx.cmp.getKeyFieldCount(), ctx.splitKey.getBuffer().array(), 0);
ctx.splitKey.getTuple().resetByTupleOffset(ctx.splitKey.getBuffer(), 0);
ctx.splitKey.setLeftPage(frontier.pageId);
@@ -920,20 +794,20 @@
// assumes btree has been created and opened
@Override
- public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws TreeIndexException, HyracksDataException, PageAllocationException {
+ public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws TreeIndexException, HyracksDataException {
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame)leafFrameFactory.createFrame();
if (!isEmptyTree(leafFrame)) {
throw new BTreeException("Trying to Bulk-load a non-empty BTree.");
}
BulkLoadContext ctx = new BulkLoadContext(fillFactor, leafFrame,
- (IBTreeInteriorFrame)interiorFrameFactory.createFrame(), freePageManager.getMetaDataFrameFactory().createFrame(), cmp);
- ctx.splitKey.getTuple().setFieldCount(cmp.getKeyFieldCount());
+ (IBTreeInteriorFrame)interiorFrameFactory.createFrame(), freePageManager.getMetaDataFrameFactory().createFrame(), cmpFactories);
+ ctx.splitKey.getTuple().setFieldCount(ctx.cmp.getKeyFieldCount());
return ctx;
}
@Override
- public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException, PageAllocationException {
+ public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException {
BulkLoadContext ctx = (BulkLoadContext) ictx;
NodeFrontier leafFrontier = ctx.nodeFrontiers.get(0);
IBTreeLeafFrame leafFrame = ctx.leafFrame;
@@ -949,13 +823,12 @@
if (spaceUsed + spaceNeeded > ctx.leafMaxBytes) {
leafFrontier.lastTuple.resetByTupleIndex(leafFrame, leafFrame.getTupleCount() - 1);
- int splitKeySize = ctx.tupleWriter.bytesRequired(leafFrontier.lastTuple, 0, cmp.getKeyFieldCount());
+ int splitKeySize = ctx.tupleWriter.bytesRequired(leafFrontier.lastTuple, 0, ctx.cmp.getKeyFieldCount());
ctx.splitKey.initData(splitKeySize);
- ctx.tupleWriter.writeTupleFields(leafFrontier.lastTuple, 0, cmp.getKeyFieldCount(),
- ctx.splitKey.getBuffer(), 0);
+ ctx.tupleWriter.writeTupleFields(leafFrontier.lastTuple, 0, ctx.cmp.getKeyFieldCount(),
+ ctx.splitKey.getBuffer().array(), 0);
ctx.splitKey.getTuple().resetByTupleOffset(ctx.splitKey.getBuffer(), 0);
ctx.splitKey.setLeftPage(leafFrontier.pageId);
- int prevPageId = leafFrontier.pageId;
leafFrontier.pageId = freePageManager.getFreePage(ctx.metaFrame);
leafFrame.setNextLeaf(leafFrontier.pageId);
@@ -970,7 +843,6 @@
leafFrontier.page.acquireWriteLatch();
leafFrame.setPage(leafFrontier.page);
leafFrame.initBuffer((byte) 0);
- leafFrame.setPrevLeaf(prevPageId);
}
leafFrame.setPage(leafFrontier.page);
@@ -1010,7 +882,7 @@
private BTreeOpContext createOpContext() {
return new BTreeOpContext(leafFrameFactory, interiorFrameFactory, freePageManager.getMetaDataFrameFactory()
- .createFrame(), cmp);
+ .createFrame(), cmpFactories);
}
public ITreeIndexFrameFactory getInteriorFrameFactory() {
@@ -1021,8 +893,8 @@
return leafFrameFactory;
}
- public MultiComparator getMultiComparator() {
- return cmp;
+ public IBinaryComparatorFactory[] getComparatorFactories() {
+ return cmpFactories;
}
public IFreePageManager getFreePageManager() {
@@ -1043,6 +915,16 @@
return IndexType.BTREE;
}
+ @Override
+ public int getFileId() {
+ return fileId;
+ }
+
+ @Override
+ public IBufferCache getBufferCache() {
+ return bufferCache;
+ }
+
public byte getTreeHeight(IBTreeLeafFrame leafFrame) throws HyracksDataException {
ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
rootNode.acquireReadLatch();
@@ -1074,15 +956,16 @@
@SuppressWarnings("rawtypes")
public String printTree(IBTreeLeafFrame leafFrame, IBTreeInteriorFrame interiorFrame, ISerializerDeserializer[] keySerdes)
throws Exception {
+ MultiComparator cmp = MultiComparator.create(cmpFactories);
byte treeHeight = getTreeHeight(leafFrame);
StringBuilder strBuilder = new StringBuilder();
- printTree(rootPage, null, false, leafFrame, interiorFrame, treeHeight, keySerdes, strBuilder);
+ printTree(rootPage, null, false, leafFrame, interiorFrame, treeHeight, keySerdes, strBuilder, cmp);
return strBuilder.toString();
}
@SuppressWarnings("rawtypes")
public void printTree(int pageId, ICachedPage parent, boolean unpin, IBTreeLeafFrame leafFrame,
- IBTreeInteriorFrame interiorFrame, byte treeHeight, ISerializerDeserializer[] keySerdes, StringBuilder strBuilder) throws Exception {
+ IBTreeInteriorFrame interiorFrame, byte treeHeight, ISerializerDeserializer[] keySerdes, StringBuilder strBuilder, MultiComparator cmp) throws Exception {
ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
node.acquireReadLatch();
try {
@@ -1110,7 +993,7 @@
if (!interiorFrame.isLeaf()) {
ArrayList<Integer> children = ((BTreeNSMInteriorFrame) (interiorFrame)).getChildren(cmp);
for (int i = 0; i < children.size(); i++) {
- printTree(children.get(i), node, i == children.size() - 1, leafFrame, interiorFrame, treeHeight, keySerdes, strBuilder);
+ printTree(children.get(i), node, i == children.size() - 1, leafFrame, interiorFrame, treeHeight, keySerdes, strBuilder, cmp);
}
} else {
node.releaseReadLatch();
@@ -1128,7 +1011,9 @@
return new BTreeAccessor(this);
}
- private class BTreeAccessor implements ITreeIndexAccessor {
+ // TODO: Class should be private. But currently we need to expose the
+ // setOpContext() API to the LSM Tree for it to work correctly.
+ public class BTreeAccessor implements ITreeIndexAccessor {
private BTree btree;
private BTreeOpContext ctx;
@@ -1138,34 +1023,59 @@
}
@Override
- public void insert(ITupleReference tuple) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ public void insert(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
ctx.reset(IndexOp.INSERT);
btree.insert(tuple, ctx);
}
@Override
- public void update(ITupleReference tuple) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ public void update(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
ctx.reset(IndexOp.UPDATE);
btree.update(tuple, ctx);
}
@Override
- public void delete(ITupleReference tuple) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ public void delete(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
ctx.reset(IndexOp.DELETE);
btree.delete(tuple, ctx);
}
-
+
@Override
- public void search(ITreeIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
- TreeIndexException, PageAllocationException {
+ public void upsert(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
+ ctx.reset(IndexOp.UPSERT);
+ btree.upsert(tuple, ctx);
+ }
+
+ @Override
+ public ITreeIndexCursor createSearchCursor() {
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
+ return new BTreeRangeSearchCursor(leafFrame, false);
+ }
+
+ @Override
+ public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
+ TreeIndexException {
ctx.reset(IndexOp.SEARCH);
- btree.search(cursor, searchPred, ctx);
+ btree.search((ITreeIndexCursor) cursor, searchPred, ctx);
}
@Override
+ public ITreeIndexCursor createDiskOrderScanCursor() {
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
+ return new TreeDiskOrderScanCursor(leafFrame);
+ }
+
+ @Override
public void diskOrderScan(ITreeIndexCursor cursor) throws HyracksDataException {
ctx.reset(IndexOp.DISKORDERSCAN);
btree.diskOrderScan(cursor, ctx);
}
+
+ // TODO: Ideally, this method should not exist. But we need it for
+ // the changing the leafFrame and leafFrameFactory of the op context for
+ // the LSM-BTree to work correctly.
+ public BTreeOpContext getOpContext() {
+ return ctx;
+ }
}
}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeOpContext.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeOpContext.java
index 07c645c..23a14eb 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeOpContext.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeOpContext.java
@@ -15,6 +15,7 @@
package edu.uci.ics.hyracks.storage.am.btree.impls;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
@@ -27,9 +28,10 @@
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
public class BTreeOpContext implements IIndexOpContext {
- private final int INIT_ARRAYLIST_SIZE = 6;
- protected ITreeIndexFrameFactory leafFrameFactory;
- protected ITreeIndexFrameFactory interiorFrameFactory;
+ private final int INIT_ARRAYLIST_SIZE = 6;
+ public MultiComparator cmp;
+ public ITreeIndexFrameFactory leafFrameFactory;
+ public ITreeIndexFrameFactory interiorFrameFactory;
public IBTreeLeafFrame leafFrame;
public IBTreeInteriorFrame interiorFrame;
public ITreeIndexMetaDataFrame metaFrame;
@@ -45,7 +47,8 @@
public boolean exceptionHandled;
public BTreeOpContext(ITreeIndexFrameFactory leafFrameFactory, ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexMetaDataFrame metaFrame, MultiComparator cmp) {
+ ITreeIndexMetaDataFrame metaFrame, IBinaryComparatorFactory[] cmpFactories) {
+ this.cmp = MultiComparator.create(cmpFactories);
this.leafFrameFactory = leafFrameFactory;
this.leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
if (leafFrame != null) {
@@ -78,7 +81,7 @@
cursorInitialState = new BTreeCursorInitialState(null);
}
} else {
- // Insert, update or delete operation.
+ // Insert, delete, update or upsert operation.
if (smPages == null) {
smPages = new IntArrayList(INIT_ARRAYLIST_SIZE, INIT_ARRAYLIST_SIZE);
}
@@ -86,7 +89,7 @@
freePages = new IntArrayList(INIT_ARRAYLIST_SIZE, INIT_ARRAYLIST_SIZE);
}
if (pred == null) {
- pred = new RangePredicate(true, null, null, true, true, null, null);
+ pred = new RangePredicate(null, null, true, true, null, null);
}
if (splitKey == null) {
splitKey = new BTreeSplitKey(leafFrame.getTupleWriter().createTupleReference());
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
index 8122314..8bf4db3 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
@@ -62,7 +62,7 @@
}
@Override
- public void close() throws Exception {
+ public void close() throws HyracksDataException {
if (page != null) {
if (exclusiveLatchNodes) {
page.releaseWriteLatch();
@@ -86,66 +86,48 @@
}
private void fetchNextLeafPage(int nextLeafPage) throws HyracksDataException {
- ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false);
- if (exclusiveLatchNodes) {
- nextLeaf.acquireWriteLatch();
- page.releaseWriteLatch();
- } else {
- nextLeaf.acquireReadLatch();
- page.releaseReadLatch();
- }
- bufferCache.unpin(page);
- page = nextLeaf;
- frame.setPage(page);
+ do {
+ ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false);
+ if (exclusiveLatchNodes) {
+ nextLeaf.acquireWriteLatch();
+ page.releaseWriteLatch();
+ } else {
+ nextLeaf.acquireReadLatch();
+ page.releaseReadLatch();
+ }
+ bufferCache.unpin(page);
+ page = nextLeaf;
+ frame.setPage(page);
+ nextLeafPage = frame.getNextLeaf();
+ } while (frame.getTupleCount() == 0 && nextLeafPage > 0);
}
@Override
- public boolean hasNext() throws Exception {
- if (pred.isForward()) {
- if (tupleIndex >= frame.getTupleCount()) {
- int nextLeafPage = frame.getNextLeaf();
- if (nextLeafPage >= 0) {
- fetchNextLeafPage(nextLeafPage);
- tupleIndex = 0;
-
- stopTupleIndex = getHighKeyIndex();
- if (stopTupleIndex < 0)
- return false;
- } else {
+ public boolean hasNext() throws HyracksDataException {
+ if (tupleIndex >= frame.getTupleCount()) {
+ int nextLeafPage = frame.getNextLeaf();
+ if (nextLeafPage >= 0) {
+ fetchNextLeafPage(nextLeafPage);
+ tupleIndex = 0;
+ stopTupleIndex = getHighKeyIndex();
+ if (stopTupleIndex < 0) {
return false;
}
- }
-
- frameTuple.resetByTupleIndex(frame, tupleIndex);
- if (highKey == null || tupleIndex <= stopTupleIndex) {
- return true;
- } else
+ } else {
return false;
+ }
+ }
+
+ frameTuple.resetByTupleIndex(frame, tupleIndex);
+ if (highKey == null || tupleIndex <= stopTupleIndex) {
+ return true;
} else {
- if (tupleIndex < 0) {
- int nextLeafPage = frame.getPrevLeaf();
- if (nextLeafPage >= 0) {
- fetchNextLeafPage(nextLeafPage);
- tupleIndex = frame.getTupleCount() - 1;
-
- stopTupleIndex = getLowKeyIndex();
- if (stopTupleIndex >= frame.getTupleCount())
- return false;
- } else {
- return false;
- }
- }
-
- frameTuple.resetByTupleIndex(frame, tupleIndex);
- if (lowKey == null || tupleIndex >= stopTupleIndex) {
- return true;
- } else
- return false;
+ return false;
}
}
-
+
@Override
- public void next() throws Exception {
+ public void next() throws HyracksDataException {
tupleIndex += tupleIndexInc;
}
@@ -216,16 +198,10 @@
} else {
highKeyFtp = FindTupleNoExactMatchPolicy.LOWER_KEY;
}
-
- if (pred.isForward()) {
- tupleIndex = getLowKeyIndex();
- stopTupleIndex = getHighKeyIndex();
- tupleIndexInc = 1;
- } else {
- tupleIndex = getHighKeyIndex();
- stopTupleIndex = getLowKeyIndex();
- tupleIndexInc = -1;
- }
+
+ tupleIndex = getLowKeyIndex();
+ stopTupleIndex = getHighKeyIndex();
+ tupleIndexInc = 1;
}
@Override
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/RangePredicate.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/RangePredicate.java
index 699f6a6..607d47b 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/RangePredicate.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/RangePredicate.java
@@ -23,7 +23,6 @@
private static final long serialVersionUID = 1L;
- protected boolean isForward = true;
protected ITupleReference lowKey = null;
protected ITupleReference highKey = null;
protected boolean lowKeyInclusive = true;
@@ -34,9 +33,8 @@
public RangePredicate() {
}
- public RangePredicate(boolean isForward, ITupleReference lowKey, ITupleReference highKey, boolean lowKeyInclusive,
+ public RangePredicate(ITupleReference lowKey, ITupleReference highKey, boolean lowKeyInclusive,
boolean highKeyInclusive, MultiComparator lowKeyCmp, MultiComparator highKeyCmp) {
- this.isForward = isForward;
this.lowKey = lowKey;
this.highKey = highKey;
this.lowKeyInclusive = lowKeyInclusive;
@@ -61,10 +59,6 @@
this.highKeyCmp = highKeyCmp;
}
- public boolean isForward() {
- return isForward;
- }
-
public ITupleReference getLowKey() {
return lowKey;
}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
index f6310af..57e3a96 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
@@ -1,6 +1,7 @@
package edu.uci.ics.hyracks.storage.am.btree.util;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
@@ -10,6 +11,7 @@
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
@@ -20,29 +22,26 @@
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
public class BTreeUtils {
- public static BTree createBTree(IBufferCache bufferCache, int btreeFileId, ITypeTraits[] typeTraits, IBinaryComparator[] cmps, BTreeLeafFrameType leafType) throws BTreeException {
- MultiComparator cmp = new MultiComparator(cmps);
+ public static BTree createBTree(IBufferCache bufferCache, IOperationCallback opCallback, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, BTreeLeafFrameType leafType) throws BTreeException {
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
- BTree btree = new BTree(bufferCache, typeTraits.length, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
+ BTree btree = new BTree(bufferCache, opCallback, typeTraits.length, cmpFactories, freePageManager, interiorFrameFactory, leafFrameFactory);
return btree;
}
- public static MultiComparator getSearchMultiComparator(MultiComparator btreeCmp, ITupleReference searchKey) {
- if (searchKey == null) {
- return btreeCmp;
+ // Creates a new MultiComparator by constructing new IBinaryComparators.
+ public static MultiComparator getSearchMultiComparator(IBinaryComparatorFactory[] cmpFactories, ITupleReference searchKey) {
+ if (searchKey == null || cmpFactories.length == searchKey.getFieldCount()) {
+ return MultiComparator.create(cmpFactories);
}
- if (btreeCmp.getKeyFieldCount() == searchKey.getFieldCount()) {
- return btreeCmp;
- }
- IBinaryComparator[] cmps = new IBinaryComparator[searchKey.getFieldCount()];
+ IBinaryComparator[] newCmps = new IBinaryComparator[searchKey.getFieldCount()];
for (int i = 0; i < searchKey.getFieldCount(); i++) {
- cmps[i] = btreeCmp.getComparators()[i];
+ newCmps[i] = cmpFactories[i].createBinaryComparator();
}
- return new MultiComparator(cmps);
+ return new MultiComparator(newCmps);
}
public static ITreeIndexFrameFactory getLeafFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory, BTreeLeafFrameType leafType) throws BTreeException {
diff --git a/hyracks/hyracks-storage-am-common/pom.xml b/hyracks/hyracks-storage-am-common/pom.xml
index 4eb0b6a..f65df2f 100644
--- a/hyracks/hyracks-storage-am-common/pom.xml
+++ b/hyracks/hyracks-storage-am-common/pom.xml
@@ -51,6 +51,6 @@
<version>0.2.1-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
- </dependency>
+ </dependency>
</dependencies>
</project>
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IFreePageManager.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IFreePageManager.java
index a7901c8..98b9a7e 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IFreePageManager.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IFreePageManager.java
@@ -3,8 +3,12 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
public interface IFreePageManager {
+ public void open(int fileId);
+
+ public void close();
+
public int getFreePage(ITreeIndexMetaDataFrame metaFrame)
- throws HyracksDataException, PageAllocationException;
+ throws HyracksDataException;
public void addFreePage(ITreeIndexMetaDataFrame metaFrame, int freePage)
throws HyracksDataException;
@@ -26,4 +30,6 @@
public boolean isMetaPage(ITreeIndexMetaDataFrame metaFrame);
public boolean isFreePage(ITreeIndexMetaDataFrame metaFrame);
+
+ public int getFirstMetadataPage();
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexAccessor.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexAccessor.java
new file mode 100644
index 0000000..202769f
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexAccessor.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.api;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+/**
+ * Client handle for performing operations (insert/delete/update/search) on an
+ * IIndex. An IIndexAccessor is not thread safe, but different IIndexAccessors
+ * can concurrently operate on the same IIndex (i.e., the IIndex must allow
+ * concurrent operations).
+ */
+public interface IIndexAccessor {
+ /**
+ * Inserts the given tuple.
+ *
+ * @param tuple
+ * Tuple to be inserted.
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ * @throws IndexException
+ * If an index-specific constraint is violated, e.g., the key
+ * already exists.
+ */
+ public void insert(ITupleReference tuple) throws HyracksDataException, IndexException;
+
+ /**
+ * Updates the tuple in the index matching the given tuple with the new
+ * contents in the given tuple.
+ *
+ * @param tuple
+ * Tuple whose match in the index is to be update with the given
+ * tuples contents.
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ * @throws IndexException
+ * If there is no matching tuple in the index.
+ */
+ public void update(ITupleReference tuple) throws HyracksDataException, IndexException;
+
+ /**
+ * Deletes the tuple in the index matching the given tuple.
+ *
+ * @param tuple
+ * Tuple to be deleted.
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ * @throws IndexException
+ * If there is no matching tuple in the index.
+ */
+ public void delete(ITupleReference tuple) throws HyracksDataException, IndexException;
+
+ /**
+ * This operation is only supported by indexes with the notion of a unique key.
+ * If tuple's key already exists, then this operation performs an update.
+ * Otherwise, it performs an insert.
+ *
+ * @param tuple
+ * Tuple to be deleted.
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ * @throws IndexException
+ * If there is no matching tuple in the index.
+ *
+ */
+ public void upsert(ITupleReference tuple) throws HyracksDataException, IndexException;
+
+ /**
+ * Creates a cursor appropriate for passing into search().
+ *
+ */
+ public IIndexCursor createSearchCursor();
+
+ /**
+ * Open the given cursor for an index search using the given predicate as
+ * search condition.
+ *
+ * @param icursor
+ * Cursor over the index entries satisfying searchPred.
+ * @param searchPred
+ * Search condition.
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ * @throws IndexException
+ */
+ public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException, IndexException;
+}
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexResultCursor.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexCursor.java
similarity index 61%
copy from hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexResultCursor.java
copy to hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexCursor.java
index fb73a65..d29fd73 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexResultCursor.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexCursor.java
@@ -13,16 +13,22 @@
* limitations under the License.
*/
-package edu.uci.ics.hyracks.storage.am.invertedindex.api;
+package edu.uci.ics.hyracks.storage.am.common.api;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-public interface IInvertedIndexResultCursor {
- public boolean hasNext();
+public interface IIndexCursor {
+ public void open(ICursorInitialState initialState,
+ ISearchPredicate searchPred) throws HyracksDataException;
- public void next();
+ public boolean hasNext() throws HyracksDataException;
+
+ public void next() throws HyracksDataException;
+
+ public void close() throws HyracksDataException;
+
+ public void reset();
public ITupleReference getTuple();
-
- public void reset(IInvertedIndexSearcher invIndexSearcher);
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IOperationCallback.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IOperationCallback.java
new file mode 100644
index 0000000..9e66b43
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IOperationCallback.java
@@ -0,0 +1,9 @@
+package edu.uci.ics.hyracks.storage.am.common.api;
+
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+public interface IOperationCallback {
+ public void pre(ITupleReference tuple);
+
+ public void post(ITupleReference tuple);
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IOperationCallbackProvider.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IOperationCallbackProvider.java
new file mode 100644
index 0000000..974ef1a
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IOperationCallbackProvider.java
@@ -0,0 +1,7 @@
+package edu.uci.ics.hyracks.storage.am.common.api;
+
+import java.io.Serializable;
+
+public interface IOperationCallbackProvider extends Serializable {
+ public IOperationCallback getOperationCallback();
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndex.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndex.java
index 46d22d1..52626cf 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndex.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndex.java
@@ -15,8 +15,7 @@
package edu.uci.ics.hyracks.storage.am.common.api;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
/**
@@ -26,88 +25,38 @@
* Users must perform operations on an ITreeIndex via an ITreeIndexAccessor.
*/
public interface ITreeIndex extends IIndex {
+ /**
+ * @return The index's leaf frame factory.
+ */
+ public ITreeIndexFrameFactory getLeafFrameFactory();
- /**
- * Creates an index accessor for performing operations on this index.
- * (insert/delete/update/search/diskorderscan). An ITreeIndexAccessor is not
- * thread safe, but different ITreeIndexAccessors can concurrently operate
- * on the same ITreeIndex
- *
- * @returns ITreeIndexAccessor A tree index accessor for this tree.
- */
- public ITreeIndexAccessor createAccessor();
+ /**
+ * @return The index's interior frame factory.
+ */
+ public ITreeIndexFrameFactory getInteriorFrameFactory();
- /**
- * Prepares the index for bulk loading, returning a bulk load context. The
- * index must be empty for bulk loading to be possible.
- *
- * @param fillFactor
- * Desired fill factor in [0, 1.0].
- * @throws HyracksDataException
- * If the BufferCache throws while un/pinning or un/latching.
- * @throws TreeIndexException
- * If the tree is not empty.
- * @throws PageAllocationException
- * @returns A new context for bulk loading, required for appending tuples.
- */
- public IIndexBulkLoadContext beginBulkLoad(float fillFactor)
- throws TreeIndexException, HyracksDataException,
- PageAllocationException;
+ /**
+ * @return The index's free page manager.
+ */
+ public IFreePageManager getFreePageManager();
- /**
- * Append a tuple to the index in the context of a bulk load.
- *
- * @param tuple
- * Tuple to be inserted.
- * @param ictx
- * Existing bulk load context.
- * @throws HyracksDataException
- * If the BufferCache throws while un/pinning or un/latching.
- * @throws PageAllocationException
- */
- public void bulkLoadAddTuple(ITupleReference tuple,
- IIndexBulkLoadContext ictx) throws HyracksDataException,
- PageAllocationException;
+ /**
+ * @return The number of fields tuples of this index have.
+ */
+ public int getFieldCount();
- /**
- * Finalize the bulk loading operation in the given context.
- *
- * @param ictx
- * Existing bulk load context to be finalized.
- * @throws HyracksDataException
- * If the BufferCache throws while un/pinning or un/latching.
- * @throws PageAllocationException
- */
- public void endBulkLoad(IIndexBulkLoadContext ictx)
- throws HyracksDataException, PageAllocationException;
+ /**
+ * @return The current root page id of this index.
+ */
+ public int getRootPageId();
- /**
- * @return The index's leaf frame factory.
- */
- public ITreeIndexFrameFactory getLeafFrameFactory();
+ /**
+ * @return The file id of this index.
+ */
+ public int getFileId();
- /**
- * @return The index's interior frame factory.
- */
- public ITreeIndexFrameFactory getInteriorFrameFactory();
-
- /**
- * @return The index's free page manager.
- */
- public IFreePageManager getFreePageManager();
-
- /**
- * @return The number of fields tuples of this index have.
- */
- public int getFieldCount();
-
- /**
- * @return The current root page id of this index.
- */
- public int getRootPageId();
-
- /**
- * @return An enum of the concrete type of this index.
- */
- public IndexType getIndexType();
+ /**
+ * @return Comparator factories.
+ */
+ public IBinaryComparatorFactory[] getComparatorFactories();
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexAccessor.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexAccessor.java
index 1e679b2..da8fc3b 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexAccessor.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexAccessor.java
@@ -16,7 +16,6 @@
package edu.uci.ics.hyracks.storage.am.common.api;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
/**
* Client handle for performing operations
@@ -25,68 +24,13 @@
* concurrently operate on the same ITreeIndex (i.e., the ITreeIndex must allow
* concurrent operations).
*/
-public interface ITreeIndexAccessor {
+public interface ITreeIndexAccessor extends IIndexAccessor {
/**
- * Inserts the given tuple.
+ * Creates a cursor appropriate for passing into diskOrderScan().
*
- * @param tuple
- * Tuple to be inserted.
- * @throws HyracksDataException
- * If the BufferCache throws while un/pinning or un/latching.
- * @throws TreeIndexException
- * If an index-specific constraint is violated, e.g., the key
- * already exists.
- * @throws PageAllocationException
*/
- public void insert(ITupleReference tuple) throws HyracksDataException,
- TreeIndexException, PageAllocationException;
-
- /**
- * Updates the tuple in the index matching the given tuple with the new
- * contents in the given tuple.
- *
- * @param tuple
- * Tuple whose match in the index is to be update with the given
- * tuples contents.
- * @throws HyracksDataException
- * If the BufferCache throws while un/pinning or un/latching.
- * @throws TreeIndexException
- * If there is no matching tuple in the index.
- * @throws PageAllocationException
- */
- public void update(ITupleReference tuple) throws HyracksDataException,
- TreeIndexException, PageAllocationException;
-
- /**
- * Deletes the tuple in the index matching the given tuple.
- *
- * @param tuple
- * Tuple to be deleted.
- * @throws HyracksDataException
- * If the BufferCache throws while un/pinning or un/latching.
- * @throws TreeIndexException
- * If there is no matching tuple in the index.
- * @throws PageAllocationException
- */
- public void delete(ITupleReference tuple) throws HyracksDataException,
- TreeIndexException, PageAllocationException;
-
- /**
- * Open the given cursor for an index search using the given predicate as
- * search condition.
- *
- * @param icursor
- * Cursor over the index entries satisfying searchPred.
- * @param searchPred
- * Search condition.
- * @throws HyracksDataException
- * If the BufferCache throws while un/pinning or un/latching.
- * @throws TreeIndexException
- * @throws PageAllocationException
- */
- public void search(ITreeIndexCursor cursor, ISearchPredicate searchPred)
- throws HyracksDataException, TreeIndexException, PageAllocationException;
-
+ public ITreeIndexCursor createDiskOrderScanCursor();
+
/**
* Open the given cursor for a disk-order scan, positioning the cursor to
* the first leaf tuple.
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexCursor.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexCursor.java
index d3ce3867..229613c 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexCursor.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexCursor.java
@@ -15,31 +15,17 @@
package edu.uci.ics.hyracks.storage.am.common.api;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
-public interface ITreeIndexCursor {
- public void reset();
-
- public boolean hasNext() throws Exception;
-
- public void next() throws Exception;
-
- public void open(ICursorInitialState initialState,
- ISearchPredicate searchPred) throws HyracksDataException;
+public interface ITreeIndexCursor extends IIndexCursor {
public ICachedPage getPage();
- public void close() throws Exception;
-
public void setBufferCache(IBufferCache bufferCache);
public void setFileId(int fileId);
- public ITupleReference getTuple();
-
// For allowing updates.
public boolean exclusiveLatchNodes();
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java
index 17519ae..9e95970 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java
@@ -41,4 +41,10 @@
public boolean hasSpace();
public void addFreePage(int freePage);
+
+ // Special flag for LSM-Components to mark whether they are valid or not.
+ public boolean isValid();
+
+ // Set special validity flag.
+ public void setValid(boolean isValid);
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexTupleWriter.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexTupleWriter.java
index f0bb7aa..30e8f39 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexTupleWriter.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexTupleWriter.java
@@ -26,8 +26,7 @@
public int bytesRequired(ITupleReference tuple);
- // TODO: change to byte[] as well.
- public int writeTupleFields(ITupleReference tuple, int startField, int numFields, ByteBuffer targetBuf,
+ public int writeTupleFields(ITupleReference tuple, int startField, int numFields, byte[] targetBuf,
int targetOff);
public int bytesRequired(ITupleReference tuple, int startField, int numFields);
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IEvaluator.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITupleFilter.java
similarity index 73%
copy from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IEvaluator.java
copy to hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITupleFilter.java
index 566651d..41f0f4d 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IEvaluator.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITupleFilter.java
@@ -12,11 +12,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
-import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+package edu.uci.ics.hyracks.storage.am.common.api;
+
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-public interface IEvaluator {
- public void evaluate(IFrameTupleReference tuple) throws AlgebricksException;
-}
\ No newline at end of file
+public interface ITupleFilter {
+ public boolean accept(IFrameTupleReference tuple) throws Exception;
+}
diff --git a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITupleFilterFactory.java
similarity index 71%
copy from hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
copy to hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITupleFilterFactory.java
index 871fe43..602902d 100644
--- a/hyracks/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITupleFilterFactory.java
@@ -12,10 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+
+package edu.uci.ics.hyracks.storage.am.common.api;
import java.io.Serializable;
-public interface IPrinterFactory extends Serializable {
- public IPrinter createPrinter();
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+
+public interface ITupleFilterFactory extends Serializable {
+ public ITupleFilter createTupleFilter(IHyracksTaskContext ctx) throws Exception;
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/PageAllocationException.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IndexException.java
similarity index 75%
rename from hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/PageAllocationException.java
rename to hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IndexException.java
index e6eec66..0aeaf82 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/PageAllocationException.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IndexException.java
@@ -15,15 +15,14 @@
package edu.uci.ics.hyracks.storage.am.common.api;
-public class PageAllocationException extends Exception {
+public class IndexException extends Exception {
+ private static final long serialVersionUID = 1L;
- private static final long serialVersionUID = 1L;
-
- public PageAllocationException(Throwable cause) {
- super(cause);
+ public IndexException(Exception e) {
+ super(e);
}
-
- public PageAllocationException(String message) {
+
+ public IndexException(String message) {
super(message);
}
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/TreeIndexException.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/TreeIndexException.java
index 4cf596f..c3f3f1a 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/TreeIndexException.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/TreeIndexException.java
@@ -15,7 +15,7 @@
package edu.uci.ics.hyracks.storage.am.common.api;
-public class TreeIndexException extends Exception {
+public class TreeIndexException extends IndexException {
private static final long serialVersionUID = 1L;
private boolean handled = false;
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/AbstractTreeIndexOperatorDescriptor.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/AbstractTreeIndexOperatorDescriptor.java
index ff4d078..9f0fbc9 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/AbstractTreeIndexOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/AbstractTreeIndexOperatorDescriptor.java
@@ -18,10 +18,11 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.ITupleFilterFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
public abstract class AbstractTreeIndexOperatorDescriptor extends
@@ -34,33 +35,36 @@
protected final IBinaryComparatorFactory[] comparatorFactories;
- protected final ITreeIndexFrameFactory interiorFrameFactory;
- protected final ITreeIndexFrameFactory leafFrameFactory;
-
protected final IStorageManagerInterface storageManager;
protected final IIndexRegistryProvider<IIndex> indexRegistryProvider;
protected final ITypeTraits[] typeTraits;
protected final IIndexDataflowHelperFactory dataflowHelperFactory;
+ protected final ITupleFilterFactory tupleFilterFactory;
+
+ protected final boolean retainInput;
+ protected final IOperationCallbackProvider opCallbackProvider;
- public AbstractTreeIndexOperatorDescriptor(JobSpecification spec,
+ public AbstractTreeIndexOperatorDescriptor(IOperatorDescriptorRegistry spec,
int inputArity, int outputArity, RecordDescriptor recDesc,
IStorageManagerInterface storageManager,
IIndexRegistryProvider<IIndex> indexRegistryProvider,
IFileSplitProvider fileSplitProvider,
- ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, ITypeTraits[] typeTraits,
+ ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] comparatorFactories,
- IIndexDataflowHelperFactory dataflowHelperFactory) {
+ IIndexDataflowHelperFactory dataflowHelperFactory,
+ ITupleFilterFactory tupleFilterFactory,
+ boolean retainInput, IOperationCallbackProvider opCallbackProvider) {
super(spec, inputArity, outputArity);
this.fileSplitProvider = fileSplitProvider;
this.storageManager = storageManager;
this.indexRegistryProvider = indexRegistryProvider;
- this.interiorFrameFactory = interiorFrameFactory;
- this.leafFrameFactory = leafFrameFactory;
this.typeTraits = typeTraits;
this.comparatorFactories = comparatorFactories;
this.dataflowHelperFactory = dataflowHelperFactory;
+ this.retainInput = retainInput;
+ this.tupleFilterFactory = tupleFilterFactory;
+ this.opCallbackProvider = opCallbackProvider;
if (outputArity > 0) {
recordDescriptors[0] = recDesc;
}
@@ -82,16 +86,6 @@
}
@Override
- public ITreeIndexFrameFactory getTreeIndexInteriorFactory() {
- return interiorFrameFactory;
- }
-
- @Override
- public ITreeIndexFrameFactory getTreeIndexLeafFactory() {
- return leafFrameFactory;
- }
-
- @Override
public IStorageManagerInterface getStorageManager() {
return storageManager;
}
@@ -110,4 +104,19 @@
public IIndexDataflowHelperFactory getIndexDataflowHelperFactory() {
return dataflowHelperFactory;
}
+
+ @Override
+ public boolean getRetainInput() {
+ return retainInput;
+ }
+
+ @Override
+ public IOperationCallbackProvider getOpCallbackProvider() {
+ return opCallbackProvider;
+ }
+
+ @Override
+ public ITupleFilterFactory getTupleFilterFactory() {
+ return tupleFilterFactory;
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndex.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndex.java
index 38b275b..64cbd58 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndex.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndex.java
@@ -16,14 +16,22 @@
package edu.uci.ics.hyracks.storage.am.common.dataflow;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
/**
- * Interface describing the operations common to all indexes.
+ * Interface describing the operations common to all index structures. Indexes
+ * implementing this interface can easily reuse existing index operators for
+ * dataflow. Users must perform operations on an IIndex via an IIndexAccessor.
*/
public interface IIndex {
/**
- * Initializes the persistent state of an index, e.g., the root page,
- * and metadata pages.
+ * Initializes the persistent state of an index, e.g., the root page, and
+ * metadata pages.
*
* @param indexFileId
* The file id to use for this index.
@@ -38,10 +46,67 @@
* @param indexFileId
* The file id backing this index.
*/
- public void open(int indexFileId);
-
+ public void open(int indexFileId) throws HyracksDataException;
+
/**
* Closes the index.
*/
- public void close();
+ public void close() throws HyracksDataException;
+
+ /**
+ * Creates an index accessor for performing operations on this index.
+ * (insert/delete/update/search/diskorderscan). An IIndexAccessor is not
+ * thread safe, but different IIndexAccessors can concurrently operate
+ * on the same IIndex
+ *
+ * @returns IIndexAccessor An accessor for this tree.
+ */
+ public IIndexAccessor createAccessor();
+
+ /**
+ * Prepares the index for bulk loading, returning a bulk load context. The
+ * index may require to be empty for bulk loading.
+ *
+ * @param fillFactor
+ * Desired fill factor in [0, 1.0].
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ * @throws IndexException
+ * For example, if the index was already loaded and only
+ * supports a single load.
+ * @returns A new context for bulk loading, required for appending tuples.
+ */
+ public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws IndexException, HyracksDataException;
+
+ /**
+ * Append a tuple to the index in the context of a bulk load.
+ *
+ * @param tuple
+ * Tuple to be inserted.
+ * @param ictx
+ * Existing bulk load context.
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ */
+ public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException;
+
+ /**
+ * Finalize the bulk loading operation in the given context.
+ *
+ * @param ictx
+ * Existing bulk load context to be finalized.
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ */
+ public void endBulkLoad(IIndexBulkLoadContext ictx) throws HyracksDataException;
+
+ /**
+ * @return BufferCache underlying this index.
+ */
+ public IBufferCache getBufferCache();
+
+ /**
+ * @return An enum of the concrete type of this index.
+ */
+ public IndexType getIndexType();
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndexDataflowHelperFactory.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndexDataflowHelperFactory.java
index a42cf5f..ddca470 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndexDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndexDataflowHelperFactory.java
@@ -21,5 +21,5 @@
public interface IIndexDataflowHelperFactory extends Serializable {
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc,
- final IHyracksTaskContext ctx, int partition, boolean createIfNotExists);
-}
+ final IHyracksTaskContext ctx, int partition);
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndexOperatorDescriptor.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndexOperatorDescriptor.java
index edf2bba..e37d374 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndexOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndexOperatorDescriptor.java
@@ -18,6 +18,7 @@
import edu.uci.ics.hyracks.api.dataflow.IActivity;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
public interface IIndexOperatorDescriptor extends IActivity {
@@ -30,4 +31,8 @@
public RecordDescriptor getRecordDescriptor();
public IIndexDataflowHelperFactory getIndexDataflowHelperFactory();
+
+ public boolean getRetainInput();
+
+ public IOperationCallbackProvider getOpCallbackProvider();
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/ITreeIndexOperatorDescriptor.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/ITreeIndexOperatorDescriptor.java
index e415cbb..7fba22b 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/ITreeIndexOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/ITreeIndexOperatorDescriptor.java
@@ -17,15 +17,12 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITupleFilterFactory;
public interface ITreeIndexOperatorDescriptor extends IIndexOperatorDescriptor {
public IBinaryComparatorFactory[] getTreeIndexComparatorFactories();
- public ITreeIndexFrameFactory getTreeIndexInteriorFactory();
-
- public ITreeIndexFrameFactory getTreeIndexLeafFactory();
-
public ITypeTraits[] getTreeIndexTypeTraits();
+ public ITupleFilterFactory getTupleFilterFactory();
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
index 99ebab7..fa95ce4 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
@@ -19,39 +19,39 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public abstract class IndexDataflowHelper {
protected IIndex index;
protected int indexFileId = -1;
- protected int partition;
-
+
+ protected final int partition;
protected final IIndexOperatorDescriptor opDesc;
protected final IHyracksTaskContext ctx;
- protected final boolean createIfNotExists;
-
- public IndexDataflowHelper(IIndexOperatorDescriptor opDesc, final IHyracksTaskContext ctx,
- int partition, boolean createIfNotExists) {
+
+ public IndexDataflowHelper(IIndexOperatorDescriptor opDesc, final IHyracksTaskContext ctx, int partition) {
this.opDesc = opDesc;
- this.ctx = ctx;
+ this.ctx = ctx;
this.partition = partition;
- this.createIfNotExists = createIfNotExists;
}
- public void init() throws HyracksDataException {
+ public void init(boolean forceCreate) throws HyracksDataException {
IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
- IFileMapProvider fileMapProvider = opDesc.getStorageManager().getFileMapProvider(ctx);
-
- FileReference f = getFilereference();
+ IFileMapProvider fileMapProvider = opDesc.getStorageManager().getFileMapProvider(ctx);
+ IndexRegistry<IIndex> indexRegistry = opDesc.getIndexRegistryProvider().getRegistry(ctx);
+ FileReference fileRef = getFilereference();
int fileId = -1;
+ boolean fileIsMapped = false;
synchronized (fileMapProvider) {
- boolean fileIsMapped = fileMapProvider.isMapped(f);
+ fileIsMapped = fileMapProvider.isMapped(fileRef);
if (!fileIsMapped) {
- bufferCache.createFile(f);
+ bufferCache.createFile(fileRef);
}
- fileId = fileMapProvider.lookupFileId(f);
+ fileId = fileMapProvider.lookupFileId(fileRef);
try {
+ // Also creates the file if it doesn't exist yet.
bufferCache.openFile(fileId);
} catch (HyracksDataException e) {
// Revert state of buffer cache since file failed to open.
@@ -61,24 +61,24 @@
throw e;
}
}
-
- // Only set indexFileId member when openFile() succeeds,
- // otherwise deinit() will try to close the file that failed to open
+ // Only set indexFileId member after openFile() succeeds.
indexFileId = fileId;
- IndexRegistry<IIndex> indexRegistry = opDesc.getIndexRegistryProvider().getRegistry(ctx);
// Create new index instance and register it.
synchronized (indexRegistry) {
// Check if the index has already been registered.
+ boolean register = false;
index = indexRegistry.get(indexFileId);
- if (index != null) {
- return;
+ if (index == null) {
+ index = createIndexInstance();
+ register = true;
}
- index = createIndexInstance();
- if (createIfNotExists) {
- index.create(indexFileId);
+ if (forceCreate) {
+ index.create(indexFileId);
}
index.open(indexFileId);
- indexRegistry.register(indexFileId, index);
+ if (register) {
+ indexRegistry.register(indexFileId, index);
+ }
}
}
@@ -88,11 +88,12 @@
IFileSplitProvider fileSplitProvider = opDesc.getFileSplitProvider();
return fileSplitProvider.getFileSplits()[partition].getLocalFile();
}
-
+
public void deinit() throws HyracksDataException {
if (indexFileId != -1) {
IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
bufferCache.closeFile(indexFileId);
+ indexFileId = -1;
}
}
@@ -111,4 +112,8 @@
public int getIndexFileId() {
return indexFileId;
}
+
+ public IOperationCallbackProvider getOpCallbackProvider() {
+ return opDesc.getOpCallbackProvider();
+ }
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexRegistry.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexRegistry.java
index 5f3b0b3..9aba0be 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexRegistry.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexRegistry.java
@@ -19,24 +19,21 @@
public class IndexRegistry<IndexType> {
- private HashMap<Integer, IndexType> map = new HashMap<Integer, IndexType>();
+ private HashMap<Integer, IndexType> map = new HashMap<Integer, IndexType>();
- public IndexType get(int fileId) {
- return map.get(fileId);
- }
+ public IndexType get(int indexId) {
+ return map.get(indexId);
+ }
- public void register(int fileId, IndexType index) {
- map.put(fileId, index);
- }
+ public void register(int indexId, IndexType index) {
+ map.put(indexId, index);
+ }
- public void unregister(int fileId) {
- try {
- map.remove(fileId);
- } catch (Exception e) {
- }
- }
+ public void unregister(int indexId) {
+ map.remove(indexId);
+ }
- public int size() {
- return map.size();
- }
+ public int size() {
+ return map.size();
+ }
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java
index af30105..0020089 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java
@@ -20,9 +20,9 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
public class TreeIndexBulkLoadOperatorDescriptor extends AbstractTreeIndexOperatorDescriptor {
@@ -32,13 +32,13 @@
private final int[] fieldPermutation;
private final float fillFactor;
- public TreeIndexBulkLoadOperatorDescriptor(JobSpecification spec, IStorageManagerInterface storageManager,
+ public TreeIndexBulkLoadOperatorDescriptor(IOperatorDescriptorRegistry spec, IStorageManagerInterface storageManager,
IIndexRegistryProvider<IIndex> indexRegistryProvider, IFileSplitProvider fileSplitProvider,
- ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
ITypeTraits[] typeTraits, IBinaryComparatorFactory[] comparatorFactories, int[] fieldPermutation,
- float fillFactor, IIndexDataflowHelperFactory dataflowHelperFactory) {
- super(spec, 1, 0, null, storageManager, indexRegistryProvider, fileSplitProvider, interiorFrameFactory,
- leafFrameFactory, typeTraits, comparatorFactories, dataflowHelperFactory);
+ float fillFactor, IIndexDataflowHelperFactory dataflowHelperFactory,
+ IOperationCallbackProvider opCallbackProvider) {
+ super(spec, 1, 0, null, storageManager, indexRegistryProvider, fileSplitProvider, typeTraits,
+ comparatorFactories, dataflowHelperFactory, null, false, opCallbackProvider);
this.fieldPermutation = fieldPermutation;
this.fillFactor = fillFactor;
}
@@ -46,7 +46,7 @@
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
- return new TreeIndexBulkLoadOperatorNodePushable(this, ctx, partition, fieldPermutation, fillFactor,
- recordDescProvider);
+ return new TreeIndexBulkLoadOperatorNodePushable(this, ctx, partition, fieldPermutation,
+ fillFactor, recordDescProvider);
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorNodePushable.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorNodePushable.java
index bc2d67e..a2d78a4 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorNodePushable.java
@@ -24,7 +24,6 @@
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
public class TreeIndexBulkLoadOperatorNodePushable extends AbstractUnaryInputSinkOperatorNodePushable {
private float fillFactor;
@@ -40,7 +39,7 @@
public TreeIndexBulkLoadOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, int[] fieldPermutation, float fillFactor, IRecordDescriptorProvider recordDescProvider) {
treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(
- opDesc, ctx, partition, true);
+ opDesc, ctx, partition);
this.fillFactor = fillFactor;
this.recordDescProvider = recordDescProvider;
tuple.setFieldPermutation(fieldPermutation);
@@ -50,10 +49,10 @@
public void open() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor opDesc = (AbstractTreeIndexOperatorDescriptor) treeIndexHelper
.getOperatorDescriptor();
- RecordDescriptor recDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getOperatorId(), 0);
+ RecordDescriptor recDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getActivityId(), 0);
accessor = new FrameTupleAccessor(treeIndexHelper.getHyracksTaskContext().getFrameSize(), recDesc);
try {
- treeIndexHelper.init();
+ treeIndexHelper.init(false);
treeIndex = (ITreeIndex) treeIndexHelper.getIndex();
treeIndex.open(treeIndexHelper.getIndexFileId());
bulkLoadCtx = treeIndex.beginBulkLoad(fillFactor);
@@ -70,11 +69,7 @@
int tupleCount = accessor.getTupleCount();
for (int i = 0; i < tupleCount; i++) {
tuple.reset(accessor, i);
- try {
- treeIndex.bulkLoadAddTuple(tuple, bulkLoadCtx);
- } catch (PageAllocationException e) {
- throw new HyracksDataException(e);
- }
+ treeIndex.bulkLoadAddTuple(tuple, bulkLoadCtx);
}
}
@@ -82,7 +77,7 @@
public void close() throws HyracksDataException {
try {
treeIndex.endBulkLoad(bulkLoadCtx);
- } catch (PageAllocationException e) {
+ } catch (Exception e) {
throw new HyracksDataException(e);
} finally {
treeIndexHelper.deinit();
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java
new file mode 100644
index 0000000..075a6a4
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.dataflow;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
+import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+
+public class TreeIndexCreateOperatorDescriptor extends AbstractTreeIndexOperatorDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ public TreeIndexCreateOperatorDescriptor(IOperatorDescriptorRegistry spec, IStorageManagerInterface storageManager,
+ IIndexRegistryProvider<IIndex> indexRegistryProvider, IFileSplitProvider fileSplitProvider,
+ ITypeTraits[] typeTraits, IBinaryComparatorFactory[] comparatorFactories,
+ IIndexDataflowHelperFactory dataflowHelperFactory, IOperationCallbackProvider opCallbackProvider) {
+ super(spec, 0, 0, null, storageManager, indexRegistryProvider, fileSplitProvider, typeTraits,
+ comparatorFactories, dataflowHelperFactory, null, false, opCallbackProvider);
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ return new TreeIndexCreateOperatorNodePushable(this, ctx, partition);
+ }
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorNodePushable.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorNodePushable.java
new file mode 100644
index 0000000..21348a0
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorNodePushable.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.dataflow;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorNodePushable;
+
+public class TreeIndexCreateOperatorNodePushable extends AbstractOperatorNodePushable {
+ protected final TreeIndexDataflowHelper treeIndexHelper;
+
+ public TreeIndexCreateOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
+ int partition) {
+ treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(
+ opDesc, ctx, partition);
+ }
+
+ @Override
+ public void deinitialize() throws HyracksDataException {
+ }
+
+ @Override
+ public int getInputArity() {
+ return 0;
+ }
+
+ @Override
+ public IFrameWriter getInputFrameWriter(int index) {
+ return null;
+ }
+
+ @Override
+ public void initialize() throws HyracksDataException {
+ try {
+ treeIndexHelper.init(true);
+ } finally {
+ treeIndexHelper.deinit();
+ }
+ }
+
+ @Override
+ public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) {
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java
index 17c1827..10d1077 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java
@@ -24,14 +24,14 @@
public abstract class TreeIndexDataflowHelper extends IndexDataflowHelper {
protected ITreeIndexOperatorDescriptor treeOpDesc;
- public TreeIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- boolean createIfNotExists) {
- super(opDesc, ctx, partition, createIfNotExists);
+
+ public TreeIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition) {
+ super(opDesc, ctx, partition);
this.treeOpDesc = (ITreeIndexOperatorDescriptor) opDesc;
}
public abstract ITreeIndex createIndexInstance() throws HyracksDataException;
-
+
public ITreeIndexCursor createDiskOrderScanCursor(ITreeIndexFrame leafFrame) throws HyracksDataException {
return new TreeDiskOrderScanCursor(leafFrame);
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java
index 8d56d37..324485e 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java
@@ -20,33 +20,26 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
-public class TreeIndexDiskOrderScanOperatorDescriptor extends
- AbstractTreeIndexOperatorDescriptor {
+public class TreeIndexDiskOrderScanOperatorDescriptor extends AbstractTreeIndexOperatorDescriptor {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 1L;
- public TreeIndexDiskOrderScanOperatorDescriptor(JobSpecification spec,
- RecordDescriptor recDesc, IStorageManagerInterface storageManager,
- IIndexRegistryProvider<IIndex> indexRegistryProvider,
- IFileSplitProvider fileSplitProvider,
- ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, ITypeTraits[] typeTraits,
- IIndexDataflowHelperFactory dataflowHelperFactory) {
- super(spec, 0, 1, recDesc, storageManager, indexRegistryProvider,
- fileSplitProvider, interiorFrameFactory, leafFrameFactory,
- typeTraits, null, dataflowHelperFactory);
- }
+ public TreeIndexDiskOrderScanOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc,
+ IStorageManagerInterface storageManager, IIndexRegistryProvider<IIndex> indexRegistryProvider,
+ IFileSplitProvider fileSplitProvider, ITypeTraits[] typeTraits,
+ IIndexDataflowHelperFactory dataflowHelperFactory, IOperationCallbackProvider opCallbackProvider) {
+ super(spec, 0, 1, recDesc, storageManager, indexRegistryProvider, fileSplitProvider, typeTraits, null,
+ dataflowHelperFactory, null, false, opCallbackProvider);
+ }
- @Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider,
- int partition, int nPartitions) {
- return new TreeIndexDiskOrderScanOperatorNodePushable(this, ctx,
- partition);
- }
+ @Override
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ return new TreeIndexDiskOrderScanOperatorNodePushable(this, ctx, partition);
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java
index d9b4d87..d02a570 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java
@@ -31,24 +31,22 @@
public class TreeIndexDiskOrderScanOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable {
private final TreeIndexDataflowHelper treeIndexHelper;
- private final ITreeIndexOperatorDescriptor opDesc;
private ITreeIndex treeIndex;
public TreeIndexDiskOrderScanOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc,
IHyracksTaskContext ctx, int partition) {
treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(
- opDesc, ctx, partition, false);
- this.opDesc = opDesc;
+ opDesc, ctx, partition);
}
@Override
public void initialize() throws HyracksDataException {
- ITreeIndexFrame cursorFrame = opDesc.getTreeIndexLeafFactory().createFrame();
- ITreeIndexCursor cursor = treeIndexHelper.createDiskOrderScanCursor(cursorFrame);
try {
- treeIndexHelper.init();
+ treeIndexHelper.init(false);
treeIndex = (ITreeIndex) treeIndexHelper.getIndex();
- ITreeIndexAccessor indexAccessor = treeIndex.createAccessor();
+ ITreeIndexFrame cursorFrame = treeIndex.getLeafFrameFactory().createFrame();
+ ITreeIndexCursor cursor = treeIndexHelper.createDiskOrderScanCursor(cursorFrame);
+ ITreeIndexAccessor indexAccessor = (ITreeIndexAccessor) treeIndex.createAccessor();
writer.open();
try {
indexAccessor.diskOrderScan(cursor);
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorDescriptor.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorDescriptor.java
index 6ce18db..7c58031 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorDescriptor.java
@@ -18,7 +18,7 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
@@ -32,7 +32,7 @@
private IIndexRegistryProvider<IIndex> treeIndexRegistryProvider;
private IFileSplitProvider fileSplitProvider;
- public TreeIndexDropOperatorDescriptor(JobSpecification spec,
+ public TreeIndexDropOperatorDescriptor(IOperatorDescriptorRegistry spec,
IStorageManagerInterface storageManager,
IIndexRegistryProvider<IIndex> treeIndexRegistryProvider,
IFileSplitProvider fileSplitProvider) {
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorNodePushable.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorNodePushable.java
index 5c96af8..5f3c0b5 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorNodePushable.java
@@ -68,7 +68,7 @@
IndexRegistry<IIndex> treeIndexRegistry = treeIndexRegistryProvider.getRegistry(ctx);
IBufferCache bufferCache = storageManager.getBufferCache(ctx);
IFileMapProvider fileMapProvider = storageManager.getFileMapProvider(ctx);
-
+
FileReference f = fileSplitProvider.getFileSplits()[partition].getLocalFile();
int indexFileId = -1;
synchronized (fileMapProvider) {
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java
index e08f5d6..a615386 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java
@@ -21,43 +21,36 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.ITupleFilterFactory;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
-public class TreeIndexInsertUpdateDeleteOperatorDescriptor extends
- AbstractTreeIndexOperatorDescriptor {
+public class TreeIndexInsertUpdateDeleteOperatorDescriptor extends AbstractTreeIndexOperatorDescriptor {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 1L;
- private final int[] fieldPermutation;
+ private final int[] fieldPermutation;
- private IndexOp op;
+ private IndexOp op;
- public TreeIndexInsertUpdateDeleteOperatorDescriptor(JobSpecification spec,
- RecordDescriptor recDesc, IStorageManagerInterface storageManager,
- IIndexRegistryProvider<IIndex> indexRegistryProvider,
- IFileSplitProvider fileSplitProvider,
- ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] comparatorFactories,
- int[] fieldPermutation, IndexOp op,
- IIndexDataflowHelperFactory dataflowHelperFactory) {
- super(spec, 1, 1, recDesc, storageManager, indexRegistryProvider,
- fileSplitProvider, interiorFrameFactory, leafFrameFactory,
- typeTraits, comparatorFactories,
- dataflowHelperFactory);
- this.fieldPermutation = fieldPermutation;
- this.op = op;
- }
+ public TreeIndexInsertUpdateDeleteOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc,
+ IStorageManagerInterface storageManager, IIndexRegistryProvider<IIndex> indexRegistryProvider,
+ IFileSplitProvider fileSplitProvider, ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] comparatorFactories, int[] fieldPermutation, IndexOp op,
+ IIndexDataflowHelperFactory dataflowHelperFactory, ITupleFilterFactory tupleFilterFactory, IOperationCallbackProvider opCallbackProvider) {
+ super(spec, 1, 1, recDesc, storageManager, indexRegistryProvider, fileSplitProvider, typeTraits,
+ comparatorFactories, dataflowHelperFactory, tupleFilterFactory, false, opCallbackProvider);
+ this.fieldPermutation = fieldPermutation;
+ this.op = op;
+ }
- @Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider,
- int partition, int nPartitions) {
- return new TreeIndexInsertUpdateDeleteOperatorNodePushable(this, ctx,
- partition, fieldPermutation, recordDescProvider, op);
- }
+ @Override
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ return new TreeIndexInsertUpdateDeleteOperatorNodePushable(this, ctx, partition, fieldPermutation,
+ recordDescProvider, op);
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorNodePushable.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorNodePushable.java
index bf78249..e05568f 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorNodePushable.java
@@ -22,9 +22,12 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITupleFilter;
+import edu.uci.ics.hyracks.storage.am.common.api.ITupleFilterFactory;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
public class TreeIndexInsertUpdateDeleteOperatorNodePushable extends AbstractUnaryInputUnaryOutputOperatorNodePushable {
@@ -33,16 +36,16 @@
private final IRecordDescriptorProvider recordDescProvider;
private final IndexOp op;
private final PermutingFrameTupleReference tuple = new PermutingFrameTupleReference();
+ private FrameTupleReference frameTuple;
private ByteBuffer writeBuffer;
- private ITreeIndexAccessor indexAccessor;
+ private IIndexAccessor indexAccessor;
+ private ITupleFilter tupleFilter;
public TreeIndexInsertUpdateDeleteOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc,
IHyracksTaskContext ctx, int partition, int[] fieldPermutation,
IRecordDescriptorProvider recordDescProvider, IndexOp op) {
- // Only create the if insert operation is an insert.
- boolean createIfNotExists = (op == IndexOp.INSERT);
treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(
- opDesc, ctx, partition, createIfNotExists);
+ opDesc, ctx, partition);
this.recordDescProvider = recordDescProvider;
this.op = op;
tuple.setFieldPermutation(fieldPermutation);
@@ -52,14 +55,19 @@
public void open() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor opDesc = (AbstractTreeIndexOperatorDescriptor) treeIndexHelper
.getOperatorDescriptor();
- RecordDescriptor inputRecDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getOperatorId(), 0);
+ RecordDescriptor inputRecDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getActivityId(), 0);
accessor = new FrameTupleAccessor(treeIndexHelper.getHyracksTaskContext().getFrameSize(), inputRecDesc);
writeBuffer = treeIndexHelper.getHyracksTaskContext().allocateFrame();
writer.open();
try {
- treeIndexHelper.init();
+ treeIndexHelper.init(false);
ITreeIndex treeIndex = (ITreeIndex) treeIndexHelper.getIndex();
indexAccessor = treeIndex.createAccessor();
+ ITupleFilterFactory tupleFilterFactory = opDesc.getTupleFilterFactory();
+ if (tupleFilterFactory != null) {
+ tupleFilter = tupleFilterFactory.createTupleFilter(treeIndexHelper.ctx);
+ frameTuple = new FrameTupleReference();
+ }
} catch (Exception e) {
// cleanup in case of failure
treeIndexHelper.deinit();
@@ -72,8 +80,14 @@
accessor.reset(buffer);
int tupleCount = accessor.getTupleCount();
for (int i = 0; i < tupleCount; i++) {
- tuple.reset(accessor, i);
try {
+ if (tupleFilter != null) {
+ frameTuple.reset(accessor, i);
+ if (!tupleFilter.accept(frameTuple)) {
+ continue;
+ }
+ }
+ tuple.reset(accessor, i);
switch (op) {
case INSERT: {
indexAccessor.insert(tuple);
@@ -83,6 +97,10 @@
indexAccessor.update(tuple);
break;
}
+ case UPSERT: {
+ indexAccessor.upsert(tuple);
+ break;
+ }
case DELETE: {
indexAccessor.delete(tuple);
break;
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexSearchOperatorNodePushable.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexSearchOperatorNodePushable.java
new file mode 100644
index 0000000..5c19483
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexSearchOperatorNodePushable.java
@@ -0,0 +1,160 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.common.dataflow;
+
+import java.io.DataOutput;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
+
+public abstract class TreeIndexSearchOperatorNodePushable extends AbstractUnaryInputUnaryOutputOperatorNodePushable {
+ protected final TreeIndexDataflowHelper treeIndexHelper;
+ protected FrameTupleAccessor accessor;
+
+ protected ByteBuffer writeBuffer;
+ protected FrameTupleAppender appender;
+ protected ArrayTupleBuilder tb;
+ protected DataOutput dos;
+
+ protected ITreeIndex treeIndex;
+ protected ISearchPredicate searchPred;
+ protected IIndexCursor cursor;
+ protected ITreeIndexFrame cursorFrame;
+ protected IIndexAccessor indexAccessor;
+
+ protected final RecordDescriptor inputRecDesc;
+ protected final boolean retainInput;
+ protected FrameTupleReference frameTuple;
+
+ public TreeIndexSearchOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
+ int partition, IRecordDescriptorProvider recordDescProvider) {
+ treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(
+ opDesc, ctx, partition);
+ this.retainInput = treeIndexHelper.getOperatorDescriptor().getRetainInput();
+ inputRecDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getActivityId(), 0);
+ }
+
+ protected abstract ISearchPredicate createSearchPredicate();
+
+ protected abstract void resetSearchPredicate(int tupleIndex);
+
+ protected IIndexCursor createCursor() {
+ return indexAccessor.createSearchCursor();
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ accessor = new FrameTupleAccessor(treeIndexHelper.getHyracksTaskContext().getFrameSize(), inputRecDesc);
+ writer.open();
+ try {
+ treeIndexHelper.init(false);
+ treeIndex = (ITreeIndex) treeIndexHelper.getIndex();
+ cursorFrame = treeIndex.getLeafFrameFactory().createFrame();
+ searchPred = createSearchPredicate();
+ writeBuffer = treeIndexHelper.getHyracksTaskContext().allocateFrame();
+ tb = new ArrayTupleBuilder(recordDesc.getFieldCount());
+ dos = tb.getDataOutput();
+ appender = new FrameTupleAppender(treeIndexHelper.getHyracksTaskContext().getFrameSize());
+ appender.reset(writeBuffer, true);
+ indexAccessor = treeIndex.createAccessor();
+ cursor = createCursor();
+ if (retainInput) {
+ frameTuple = new FrameTupleReference();
+ }
+ } catch (Exception e) {
+ treeIndexHelper.deinit();
+ throw new HyracksDataException(e);
+ }
+ }
+
+ protected void writeSearchResults(int tupleIndex) throws Exception {
+ while (cursor.hasNext()) {
+ tb.reset();
+ cursor.next();
+ if (retainInput) {
+ frameTuple.reset(accessor, tupleIndex);
+ for (int i = 0; i < frameTuple.getFieldCount(); i++) {
+ dos.write(frameTuple.getFieldData(i), frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
+ tb.addFieldEndOffset();
+ }
+ }
+ ITupleReference tuple = cursor.getTuple();
+ for (int i = 0; i < tuple.getFieldCount(); i++) {
+ dos.write(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
+ tb.addFieldEndOffset();
+ }
+ if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ FrameUtils.flushFrame(writeBuffer, writer);
+ appender.reset(writeBuffer, true);
+ if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ accessor.reset(buffer);
+ int tupleCount = accessor.getTupleCount();
+ try {
+ for (int i = 0; i < tupleCount; i++) {
+ resetSearchPredicate(i);
+ cursor.reset();
+ indexAccessor.search(cursor, searchPred);
+ writeSearchResults(i);
+ }
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ try {
+ if (appender.getTupleCount() > 0) {
+ FrameUtils.flushFrame(writeBuffer, writer);
+ }
+ writer.close();
+ try {
+ cursor.close();
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ } finally {
+ treeIndexHelper.deinit();
+ }
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java
index 4d718a8..6bf0983 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java
@@ -21,10 +21,10 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
public class TreeIndexStatsOperatorDescriptor extends AbstractTreeIndexOperatorDescriptor {
@@ -33,13 +33,12 @@
private static final RecordDescriptor recDesc = new RecordDescriptor(
new ISerializerDeserializer[] { UTF8StringSerializerDeserializer.INSTANCE });
- public TreeIndexStatsOperatorDescriptor(JobSpecification spec, IStorageManagerInterface storageManager,
+ public TreeIndexStatsOperatorDescriptor(IOperatorDescriptorRegistry spec, IStorageManagerInterface storageManager,
IIndexRegistryProvider<IIndex> indexRegistryProvider, IFileSplitProvider fileSplitProvider,
- ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
ITypeTraits[] typeTraits, IBinaryComparatorFactory[] comparatorFactories,
- IIndexDataflowHelperFactory dataflowHelperFactory) {
- super(spec, 0, 1, recDesc, storageManager, indexRegistryProvider, fileSplitProvider, interiorFrameFactory,
- leafFrameFactory, typeTraits, comparatorFactories, dataflowHelperFactory);
+ IIndexDataflowHelperFactory dataflowHelperFactory, IOperationCallbackProvider opCallbackProvider) {
+ super(spec, 0, 1, recDesc, storageManager, indexRegistryProvider, fileSplitProvider, typeTraits,
+ comparatorFactories, dataflowHelperFactory, null, false, opCallbackProvider);
}
@Override
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
index ff007af..50486f2 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
@@ -38,7 +38,7 @@
public TreeIndexStatsOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition) {
treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(
- opDesc, ctx, partition, false);
+ opDesc, ctx, partition);
this.ctx = ctx;
}
@@ -55,7 +55,7 @@
public void initialize() throws HyracksDataException {
try {
writer.open();
- treeIndexHelper.init();
+ treeIndexHelper.init(false);
ITreeIndex treeIndex = (ITreeIndex) treeIndexHelper.getIndex();
IBufferCache bufferCache = treeIndexHelper.getOperatorDescriptor().getStorageManager().getBufferCache(ctx);
statsGatherer = new TreeIndexStatsGatherer(bufferCache, treeIndex.getFreePageManager(),
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenThread.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenThread.java
new file mode 100644
index 0000000..e8d3d56
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenThread.java
@@ -0,0 +1,71 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+
+/**
+ * Quick & dirty data generator for multi-thread testing.
+ *
+ */
+@SuppressWarnings("rawtypes")
+public class DataGenThread extends Thread {
+ public final BlockingQueue<TupleBatch> tupleBatchQueue;
+ private final int maxNumBatches;
+ private final int maxOutstandingBatches;
+ private int numBatches = 0;
+ private final Random rnd;
+
+ // maxOutstandingBatches pre-created tuple-batches for populating the queue.
+ private TupleBatch[] tupleBatches;
+ private int ringPos;
+
+ public DataGenThread(int numConsumers, int maxNumBatches, int batchSize, ISerializerDeserializer[] fieldSerdes, int payloadSize, int rndSeed, int maxOutstandingBatches, boolean sorted) {
+ this.maxNumBatches = maxNumBatches;
+ this.maxOutstandingBatches = maxOutstandingBatches;
+ rnd = new Random(rndSeed);
+ tupleBatches = new TupleBatch[maxOutstandingBatches];
+ IFieldValueGenerator[] fieldGens = DataGenUtils.getFieldGensFromSerdes(fieldSerdes, rnd, sorted);
+ for (int i = 0; i < maxOutstandingBatches; i++) {
+ tupleBatches[i] = new TupleBatch(batchSize, fieldGens, fieldSerdes, payloadSize);
+ }
+ tupleBatchQueue = new LinkedBlockingQueue<TupleBatch>(maxOutstandingBatches);
+ ringPos = 0;
+ }
+
+ @Override
+ public void run() {
+ while(numBatches < maxNumBatches) {
+ boolean added = false;
+ try {
+ if (tupleBatches[ringPos].inUse.compareAndSet(false, true)) {
+ tupleBatches[ringPos].generate();
+ tupleBatchQueue.put(tupleBatches[ringPos]);
+ added = true;
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ if (added) {
+ numBatches++;
+ ringPos++;
+ if (ringPos >= maxOutstandingBatches) {
+ ringPos = 0;
+ }
+ }
+ }
+ }
+
+ public TupleBatch getBatch() throws InterruptedException {
+ return tupleBatchQueue.take();
+ }
+
+ public void releaseBatch(TupleBatch batch) {
+ batch.inUse.set(false);
+ }
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenUtils.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenUtils.java
new file mode 100644
index 0000000..fdbaa3e
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DataGenUtils.java
@@ -0,0 +1,46 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+import java.util.Random;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+
+@SuppressWarnings("rawtypes")
+public class DataGenUtils {
+ public static IFieldValueGenerator getFieldGenFromSerde(ISerializerDeserializer serde, Random rnd, boolean sorted) {
+ if (serde instanceof IntegerSerializerDeserializer) {
+ if (sorted) {
+ return new SortedIntegerFieldValueGenerator();
+ } else {
+ return new IntegerFieldValueGenerator(rnd);
+ }
+ } else if (serde instanceof FloatSerializerDeserializer) {
+ if (sorted) {
+ return new SortedFloatFieldValueGenerator();
+ } else {
+ return new FloatFieldValueGenerator(rnd);
+ }
+ } else if (serde instanceof DoubleSerializerDeserializer) {
+ if (sorted) {
+ return new SortedDoubleFieldValueGenerator();
+ } else {
+ return new DoubleFieldValueGenerator(rnd);
+ }
+ } else if (serde instanceof UTF8StringSerializerDeserializer) {
+ return new StringFieldValueGenerator(20, rnd);
+ }
+ System.out.println("NULL");
+ return null;
+ }
+
+ public static IFieldValueGenerator[] getFieldGensFromSerdes(ISerializerDeserializer[] serdes, Random rnd, boolean sorted) {
+ IFieldValueGenerator[] fieldValueGens = new IFieldValueGenerator[serdes.length];
+ for (int i = 0; i < serdes.length; i++) {
+ fieldValueGens[i] = getFieldGenFromSerde(serdes[i], rnd, sorted);
+ }
+ return fieldValueGens;
+ }
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DoubleFieldValueGenerator.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DoubleFieldValueGenerator.java
new file mode 100644
index 0000000..fcac93a
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/DoubleFieldValueGenerator.java
@@ -0,0 +1,16 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+import java.util.Random;
+
+public class DoubleFieldValueGenerator implements IFieldValueGenerator<Double> {
+ protected final Random rnd;
+
+ public DoubleFieldValueGenerator(Random rnd) {
+ this.rnd = rnd;
+ }
+
+ @Override
+ public Double next() {
+ return rnd.nextDouble();
+ }
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/FloatFieldValueGenerator.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/FloatFieldValueGenerator.java
new file mode 100644
index 0000000..6f21c77
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/FloatFieldValueGenerator.java
@@ -0,0 +1,16 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+import java.util.Random;
+
+public class FloatFieldValueGenerator implements IFieldValueGenerator<Float> {
+ protected final Random rnd;
+
+ public FloatFieldValueGenerator(Random rnd) {
+ this.rnd = rnd;
+ }
+
+ @Override
+ public Float next() {
+ return rnd.nextFloat();
+ }
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/IFieldValueGenerator.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/IFieldValueGenerator.java
new file mode 100644
index 0000000..ee0d30b
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/IFieldValueGenerator.java
@@ -0,0 +1,5 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+public interface IFieldValueGenerator<T> {
+ public T next();
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java
new file mode 100644
index 0000000..134b1f7
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java
@@ -0,0 +1,16 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+import java.util.Random;
+
+public class IntegerFieldValueGenerator implements IFieldValueGenerator<Integer> {
+ protected final Random rnd;
+
+ public IntegerFieldValueGenerator(Random rnd) {
+ this.rnd = rnd;
+ }
+
+ @Override
+ public Integer next() {
+ return rnd.nextInt();
+ }
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/SortedDoubleFieldValueGenerator.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/SortedDoubleFieldValueGenerator.java
new file mode 100644
index 0000000..4193811
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/SortedDoubleFieldValueGenerator.java
@@ -0,0 +1,17 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+public class SortedDoubleFieldValueGenerator implements IFieldValueGenerator<Double> {
+ private double val = 0.0d;
+
+ public SortedDoubleFieldValueGenerator() {
+ }
+
+ public SortedDoubleFieldValueGenerator(double startVal) {
+ val = startVal;
+ }
+
+ @Override
+ public Double next() {
+ return val++;
+ }
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/SortedFloatFieldValueGenerator.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/SortedFloatFieldValueGenerator.java
new file mode 100644
index 0000000..1f6b315
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/SortedFloatFieldValueGenerator.java
@@ -0,0 +1,17 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+public class SortedFloatFieldValueGenerator implements IFieldValueGenerator<Float> {
+ private float val = 0.0f;
+
+ public SortedFloatFieldValueGenerator() {
+ }
+
+ public SortedFloatFieldValueGenerator(float startVal) {
+ val = startVal;
+ }
+
+ @Override
+ public Float next() {
+ return val++;
+ }
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java
new file mode 100644
index 0000000..8f7fdcf
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java
@@ -0,0 +1,17 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+public class SortedIntegerFieldValueGenerator implements IFieldValueGenerator<Integer> {
+ private int val = 0;
+
+ public SortedIntegerFieldValueGenerator() {
+ }
+
+ public SortedIntegerFieldValueGenerator(int startVal) {
+ val = startVal;
+ }
+
+ @Override
+ public Integer next() {
+ return val++;
+ }
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/StringFieldValueGenerator.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/StringFieldValueGenerator.java
new file mode 100644
index 0000000..0218542
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/StringFieldValueGenerator.java
@@ -0,0 +1,27 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+import java.util.Random;
+
+public class StringFieldValueGenerator implements IFieldValueGenerator<String> {
+ private int maxLen;
+ private final Random rnd;
+
+ public StringFieldValueGenerator(int maxLen, Random rnd) {
+ this.maxLen = maxLen;
+ this.rnd = rnd;
+ }
+
+ public void setMaxLength(int maxLen) {
+ this.maxLen = maxLen;
+ }
+
+ @Override
+ public String next() {
+ String s = Long.toHexString(Double.doubleToLongBits(rnd.nextDouble()));
+ StringBuilder strBuilder = new StringBuilder();
+ for (int i = 0; i < s.length() && i < maxLen; i++) {
+ strBuilder.append(s.charAt(Math.abs(rnd.nextInt()) % s.length()));
+ }
+ return strBuilder.toString();
+ }
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/TupleBatch.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/TupleBatch.java
new file mode 100644
index 0000000..bfa523f
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/TupleBatch.java
@@ -0,0 +1,36 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+@SuppressWarnings("rawtypes")
+public class TupleBatch {
+ private final int size;
+ private final TupleGenerator[] tupleGens;
+ public final AtomicBoolean inUse = new AtomicBoolean(false);
+
+ public TupleBatch(int size, IFieldValueGenerator[] fieldGens, ISerializerDeserializer[] fieldSerdes, int payloadSize) {
+ this.size = size;
+ tupleGens = new TupleGenerator[size];
+ for (int i = 0; i < size; i++) {
+ tupleGens[i] = new TupleGenerator(fieldGens, fieldSerdes, payloadSize);
+ }
+ }
+
+ public void generate() throws IOException {
+ for(TupleGenerator tupleGen : tupleGens) {
+ tupleGen.next();
+ }
+ }
+
+ public int size() {
+ return size;
+ }
+
+ public ITupleReference get(int ix) {
+ return tupleGens[ix].get();
+ }
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/TupleGenerator.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/TupleGenerator.java
new file mode 100644
index 0000000..2801205
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/datagen/TupleGenerator.java
@@ -0,0 +1,51 @@
+package edu.uci.ics.hyracks.storage.am.common.datagen;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+@SuppressWarnings({"rawtypes", "unchecked" })
+public class TupleGenerator {
+ protected final ISerializerDeserializer[] fieldSerdes;
+ protected final IFieldValueGenerator[] fieldGens;
+ protected final ArrayTupleBuilder tb;
+ protected final ArrayTupleReference tuple;
+ protected final byte[] payload;
+ protected final DataOutput tbDos;
+
+ public TupleGenerator(IFieldValueGenerator[] fieldGens, ISerializerDeserializer[] fieldSerdes, int payloadSize) {
+ this.fieldSerdes = fieldSerdes;
+ this.fieldGens = fieldGens;
+ tuple = new ArrayTupleReference();
+ if (payloadSize > 0) {
+ tb = new ArrayTupleBuilder(fieldSerdes.length + 1);
+ payload = new byte[payloadSize];
+ } else {
+ tb = new ArrayTupleBuilder(fieldSerdes.length);
+ payload = null;
+ }
+ tbDos = tb.getDataOutput();
+ }
+
+ public ITupleReference next() throws IOException {
+ tb.reset();
+ for (int i = 0; i < fieldSerdes.length; i++) {
+ fieldSerdes[i].serialize(fieldGens[i].next(), tbDos);
+ tb.addFieldEndOffset();
+ }
+ if (payload != null) {
+ tbDos.write(payload);
+ tb.addFieldEndOffset();
+ }
+ tuple.reset(tb.getFieldEndOffsets(), tb.getByteArray());
+ return tuple;
+ }
+
+ public ITupleReference get() {
+ return tuple;
+ }
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java
index c87b84a..31c674d 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java
@@ -27,11 +27,15 @@
public class LIFOMetaDataFrame implements ITreeIndexMetaDataFrame {
+ // Arbitrarily chosen magic integer.
+ protected static final int MAGIC_VALID_INT = 0x5bd1e995;
+
protected static final int tupleCountOff = 0; //0
protected static final int freeSpaceOff = tupleCountOff + 4; //4
protected static final int maxPageOff = freeSpaceOff + 4; //8
protected static final int levelOff = maxPageOff + 12; //20
protected static final int nextPageOff = levelOff + 1; // 21
+ protected static final int validOff = nextPageOff + 4; // 25
protected ICachedPage page = null;
protected ByteBuffer buf = null;
@@ -65,7 +69,7 @@
return buf.getInt(freeSpaceOff) + 4 < buf.capacity();
}
- // on bounds checking is done, there must be free space
+ // no bounds checking is done, there must be free space
public void addFreePage(int freePage) {
int freeSpace = buf.getInt(freeSpaceOff);
buf.putInt(freeSpace, freePage);
@@ -97,10 +101,11 @@
@Override
public void initBuffer(byte level) {
buf.putInt(tupleCountOff, 0);
- buf.putInt(freeSpaceOff, nextPageOff + 4);
+ buf.putInt(freeSpaceOff, validOff + 4);
//buf.putInt(maxPageOff, -1);
buf.put(levelOff, level);
buf.putInt(nextPageOff, -1);
+ setValid(false);
}
@Override
@@ -112,4 +117,18 @@
public void setNextPage(int nextPage) {
buf.putInt(nextPageOff, nextPage);
}
+
+ @Override
+ public boolean isValid() {
+ return buf.getInt(validOff) == MAGIC_VALID_INT;
+ }
+
+ @Override
+ public void setValid(boolean isValid) {
+ if (isValid) {
+ buf.putInt(validOff, MAGIC_VALID_INT);
+ } else {
+ buf.putInt(validOff, 0);
+ }
+ }
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManager.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManager.java
index f7e2e3b..036aa09 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManager.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManager.java
@@ -13,14 +13,13 @@
private static final byte META_PAGE_LEVEL_INDICATOR = -1;
private static final byte FREE_PAGE_LEVEL_INDICATOR = -2;
private final IBufferCache bufferCache;
- private final int fileId;
- private final int headPage;
+ private final int headPage;
+ private int fileId = -1;
private final ITreeIndexMetaDataFrameFactory metaDataFrameFactory;
- public LinkedListFreePageManager(IBufferCache bufferCache, int fileId,
+ public LinkedListFreePageManager(IBufferCache bufferCache,
int headPage, ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
this.bufferCache = bufferCache;
- this.fileId = fileId;
this.headPage = headPage;
this.metaDataFrameFactory = metaDataFrameFactory;
}
@@ -199,4 +198,19 @@
public boolean isMetaPage(ITreeIndexMetaDataFrame metaFrame) {
return metaFrame.getLevel() == META_PAGE_LEVEL_INDICATOR;
}
+
+ @Override
+ public int getFirstMetadataPage() {
+ return headPage;
+ }
+
+ @Override
+ public void open(int fileId) {
+ this.fileId = fileId;
+ }
+
+ @Override
+ public void close() {
+ fileId = -1;
+ }
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java
new file mode 100644
index 0000000..157b563
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.freepage;
+
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+public class LinkedListFreePageManagerFactory {
+
+ private final ITreeIndexMetaDataFrameFactory metaDataFrameFactory;
+ private final IBufferCache bufferCache;
+
+ public LinkedListFreePageManagerFactory(IBufferCache bufferCache, ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
+ this.metaDataFrameFactory = metaDataFrameFactory;
+ this.bufferCache = bufferCache;
+ }
+
+ public IFreePageManager createFreePageManager() {
+ return new LinkedListFreePageManager(bufferCache, 0, metaDataFrameFactory);
+ }
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/NoOpOperationCallback.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/NoOpOperationCallback.java
new file mode 100644
index 0000000..828dd81
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/NoOpOperationCallback.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.impls;
+
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallback;
+
+/**
+ * Dummy operation callback that simply does nothing. Mainly, intended to be
+ * used in non-transaction access method testing.
+ */
+public class NoOpOperationCallback implements IOperationCallback {
+
+ public static IOperationCallback INSTANCE = new NoOpOperationCallback();
+
+ private NoOpOperationCallback() {
+ }
+
+ @Override
+ public void pre(ITupleReference tuple) {
+ // Do nothing.
+ }
+
+ @Override
+ public void post(ITupleReference tuple) {
+ // Do nothing.
+ }
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/NoOpOperationCallbackProvider.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/NoOpOperationCallbackProvider.java
new file mode 100644
index 0000000..55dfb74e
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/NoOpOperationCallbackProvider.java
@@ -0,0 +1,19 @@
+package edu.uci.ics.hyracks.storage.am.common.impls;
+
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
+
+/**
+ * Dummy NoOp callback provider used primarily for testing. Always returns the
+ * {@link NoOpOperationCallback} instance.
+ *
+ * Implemented as an enum to preserve singleton model while being serializable
+ */
+public enum NoOpOperationCallbackProvider implements IOperationCallbackProvider {
+ INSTANCE;
+
+ @Override
+ public IOperationCallback getOperationCallback() {
+ return NoOpOperationCallback.INSTANCE;
+ }
+}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/TreeDiskOrderScanCursor.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/TreeDiskOrderScanCursor.java
index 1a81c3f..ea4c105 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/TreeDiskOrderScanCursor.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/TreeDiskOrderScanCursor.java
@@ -43,7 +43,7 @@
}
@Override
- public void close() throws Exception {
+ public void close() throws HyracksDataException {
page.releaseReadLatch();
bufferCache.unpin(page);
page = null;
@@ -61,30 +61,31 @@
private boolean positionToNextLeaf(boolean skipCurrent)
throws HyracksDataException {
- while ((frame.getLevel() != 0 || skipCurrent) && (currentPageId <= maxPageId)) {
+ while ((frame.getLevel() != 0 || skipCurrent || frame.getTupleCount() == 0) && (currentPageId <= maxPageId)) {
currentPageId++;
+ page.releaseReadLatch();
+ bufferCache.unpin(page);
+
ICachedPage nextPage = bufferCache.pin(
BufferedFileHandle.getDiskPageId(fileId, currentPageId),
false);
nextPage.acquireReadLatch();
- page.releaseReadLatch();
- bufferCache.unpin(page);
-
page = nextPage;
frame.setPage(page);
tupleIndex = 0;
skipCurrent = false;
}
- if (currentPageId <= maxPageId)
+ if (currentPageId <= maxPageId) {
return true;
- else
+ } else {
return false;
+ }
}
@Override
- public boolean hasNext() throws Exception {
+ public boolean hasNext() throws HyracksDataException {
if (currentPageId > maxPageId) {
return false;
}
@@ -102,7 +103,7 @@
}
@Override
- public void next() throws Exception {
+ public void next() throws HyracksDataException {
tupleIndex++;
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/IndexOp.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/IndexOp.java
index 780acd8..77ad7ff 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/IndexOp.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/IndexOp.java
@@ -16,5 +16,5 @@
package edu.uci.ics.hyracks.storage.am.common.ophelpers;
public enum IndexOp {
- INSERT, DELETE, UPDATE, SEARCH, DISKORDERSCAN
+ INSERT, DELETE, UPDATE, UPSERT, SEARCH, DISKORDERSCAN, PHYSICALDELETE
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/IntArrayList.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/IntArrayList.java
index d888aa0..12bc997 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/IntArrayList.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/IntArrayList.java
@@ -16,74 +16,83 @@
package edu.uci.ics.hyracks.storage.am.common.ophelpers;
public class IntArrayList {
- private int[] data;
- private int size;
- private int first;
- private final int growth;
+ private int[] data;
+ private int size;
+ private int first;
+ private final int growth;
- public IntArrayList(int initialCapacity, int growth) {
- data = new int[initialCapacity];
- size = 0;
- first = 0;
- this.growth = growth;
- }
+ public IntArrayList(int initialCapacity, int growth) {
+ data = new int[initialCapacity];
+ size = 0;
+ first = 0;
+ this.growth = growth;
+ }
- public int size() {
- return size;
- }
+ public int size() {
+ return size;
+ }
- public int first() {
- return first;
- }
+ public int first() {
+ return first;
+ }
- public void add(int i) {
- if (size == data.length) {
- int[] newData = new int[data.length + growth];
- System.arraycopy(data, 0, newData, 0, data.length);
- data = newData;
- }
+ public void add(int i) {
+ if (size == data.length) {
+ int[] newData = new int[data.length + growth];
+ System.arraycopy(data, 0, newData, 0, data.length);
+ data = newData;
+ }
- data[size++] = i;
- }
+ data[size++] = i;
+ }
- public void removeLast() {
- if (size > 0)
- size--;
- }
+ public void addFirst(int i) {
+ int[] newData = new int[data.length + 1];
+ System.arraycopy(data, 0, newData, 0, first);
+ System.arraycopy(data, first, newData, first + 1, size - first);
+ data = newData;
+ data[first] = i;
+ size++;
+ }
- // WARNING: caller is responsible for checking size > 0
- public int getLast() {
- return data[size - 1];
- }
+ public void removeLast() {
+ if (size > 0)
+ size--;
+ }
- public int get(int i) {
- return data[i];
- }
+ // WARNING: caller is responsible for checking size > 0
+ public int getLast() {
+ return data[size - 1];
+ }
- // WARNING: caller is responsible for checking i < size
- public void set(int i, int value) {
- data[i] = value;
+ public int get(int i) {
+ return data[i];
+ }
- }
+ // WARNING: caller is responsible for checking i < size
+ public void set(int i, int value) {
+ data[i] = value;
- public int getFirst() {
- return data[first];
- }
+ }
- public void moveFirst() {
- first++;
- }
+ public int getFirst() {
+ return data[first];
+ }
- public void clear() {
- size = 0;
- first = 0;
- }
+ public void moveFirst() {
+ first++;
+ }
- public boolean isLast() {
- return size == first;
- }
+ public void clear() {
+ size = 0;
+ first = 0;
+ }
- public boolean isEmpty() {
- return size == 0;
- }
+ public boolean isLast() {
+ return size == first;
+ }
+
+ public boolean isEmpty() {
+ return size == 0;
+ }
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/LongArrayList.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/LongArrayList.java
index 4dd1b5f..cb4c8fe 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/LongArrayList.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/LongArrayList.java
@@ -35,6 +35,16 @@
public int first() {
return first;
}
+
+ public void addFirst(long i) {
+ long[] newData = new long[data.length + 1];
+ System.arraycopy(data, 0, newData, 0, first);
+ System.arraycopy(data, first, newData, first + 1, size - first);
+ data = newData;
+ data[first] = i;
+ size++;
+ }
+
public void add(long i) {
if (size == data.length) {
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/MultiComparator.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/MultiComparator.java
index df24484..c653c9a 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/MultiComparator.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/MultiComparator.java
@@ -16,6 +16,7 @@
package edu.uci.ics.hyracks.storage.am.common.ophelpers;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
public class MultiComparator {
@@ -54,12 +55,36 @@
}
return 0;
}
+
+ public int compare(ITupleReference tupleA,
+ ITupleReference tupleB, int startFieldIndex) {
+ for (int i = 0; i < cmps.length; i++) {
+ int ix = startFieldIndex + i;
+ int cmp = cmps[i].compare(tupleA.getFieldData(ix),
+ tupleA.getFieldStart(ix), tupleA.getFieldLength(ix),
+ tupleB.getFieldData(ix), tupleB.getFieldStart(ix),
+ tupleB.getFieldLength(ix));
+ if (cmp < 0)
+ return -1;
+ else if (cmp > 0)
+ return 1;
+ }
+ return 0;
+ }
public IBinaryComparator[] getComparators() {
return cmps;
}
- public int getKeyFieldCount() {
+ public int getKeyFieldCount() {
return cmps.length;
}
+
+ public static MultiComparator create(IBinaryComparatorFactory[] cmpFactories) {
+ IBinaryComparator[] cmps = new IBinaryComparator[cmpFactories.length];
+ for (int i = 0; i < cmpFactories.length; i++) {
+ cmps[i] = cmpFactories[i].createBinaryComparator();
+ }
+ return new MultiComparator(cmps);
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/SimpleTupleWriter.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/SimpleTupleWriter.java
index 831247e..f5ec5f3 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/SimpleTupleWriter.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/SimpleTupleWriter.java
@@ -29,12 +29,6 @@
buf[targetOff + 1] = (byte)(s >> 0);
}
- // Write short in big endian to target byte array at given offset.
- private static void writeShortB(short s, byte[] buf, int targetOff) {
- buf[targetOff] = (byte) (s >> 0);
- buf[targetOff + 1] = (byte) (s >> 8);
- }
-
@Override
public int bytesRequired(ITupleReference tuple) {
int bytes = getNullFlagsBytes(tuple) + getFieldSlotsBytes(tuple);
@@ -84,23 +78,23 @@
}
@Override
- public int writeTupleFields(ITupleReference tuple, int startField, int numFields, ByteBuffer targetBuf,
+ public int writeTupleFields(ITupleReference tuple, int startField, int numFields, byte[] targetBuf,
int targetOff) {
int runner = targetOff;
int nullFlagsBytes = getNullFlagsBytes(tuple, startField, numFields);
for (int i = 0; i < nullFlagsBytes; i++) {
- targetBuf.put(runner++, (byte) 0);
+ targetBuf[runner++] = (byte) 0;
}
runner += getFieldSlotsBytes(tuple, startField, numFields);
int fieldEndOff = 0;
int fieldCounter = 0;
for (int i = startField; i < startField + numFields; i++) {
- System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf.array(), runner,
+ System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf, runner,
tuple.getFieldLength(i));
fieldEndOff += tuple.getFieldLength(i);
- runner += tuple.getFieldLength(i);
- targetBuf.putShort(targetOff + nullFlagsBytes + fieldCounter * 2, (short) fieldEndOff);
+ runner += tuple.getFieldLength(i);
+ writeShortL((short) fieldEndOff, targetBuf, targetOff + nullFlagsBytes + fieldCounter * 2);
fieldCounter++;
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java
index fe52608..9730346 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java
@@ -79,8 +79,6 @@
// write data fields
for (int i = 0; i < tuple.getFieldCount(); i++) {
- int s = tuple.getFieldStart(i);
- int l = tuple.getFieldLength(i);
System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf, runner, tuple.getFieldLength(i));
runner += tuple.getFieldLength(i);
}
@@ -89,17 +87,17 @@
}
@Override
- public int writeTupleFields(ITupleReference tuple, int startField, int numFields, ByteBuffer targetBuf,
+ public int writeTupleFields(ITupleReference tuple, int startField, int numFields, byte[] targetBuf,
int targetOff) {
int runner = targetOff;
int nullFlagsBytes = getNullFlagsBytes(numFields);
// write null indicator bits
for (int i = 0; i < nullFlagsBytes; i++) {
- targetBuf.put(runner++, (byte) 0);
+ targetBuf[runner++] = (byte) 0;
}
// write field slots for variable length fields
- encDec.reset(targetBuf.array(), runner);
+ encDec.reset(targetBuf, runner);
for (int i = startField; i < startField + numFields; i++) {
if (!typeTraits[i].isFixedLength()) {
encDec.encode(tuple.getFieldLength(i));
@@ -108,7 +106,7 @@
runner = encDec.getPos();
for (int i = startField; i < startField + numFields; i++) {
- System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf.array(), runner,
+ System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf, runner,
tuple.getFieldLength(i));
runner += tuple.getFieldLength(i);
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/IndexUtils.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/IndexUtils.java
deleted file mode 100644
index 389855f..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/IndexUtils.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.storage.am.common.util;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-
-public class IndexUtils {
- public static MultiComparator createMultiComparator(IBinaryComparatorFactory[] cmpFactories) {
- IBinaryComparator[] cmps = new IBinaryComparator[cmpFactories.length];
- for (int i = 0; i < cmpFactories.length; i++) {
- cmps[i] = cmpFactories[i].createBinaryComparator();
- }
- return new MultiComparator(cmps);
- }
-}
diff --git a/hyracks/hyracks-storage-am-invertedindex/pom.xml b/hyracks/hyracks-storage-am-invertedindex/pom.xml
index 231860b..45add2a 100644
--- a/hyracks/hyracks-storage-am-invertedindex/pom.xml
+++ b/hyracks/hyracks-storage-am-invertedindex/pom.xml
@@ -47,13 +47,6 @@
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
- <artifactId>hyracks-control-nc</artifactId>
- <version>0.2.1-SNAPSHOT</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-btree</artifactId>
<version>0.2.1-SNAPSHOT</version>
<type>jar</type>
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexOperatorDescriptor.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexOperatorDescriptor.java
index 1dc753e..042042f 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexOperatorDescriptor.java
@@ -19,10 +19,13 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizerFactory;
public interface IInvertedIndexOperatorDescriptor extends ITreeIndexOperatorDescriptor {
public IBinaryComparatorFactory[] getInvListsComparatorFactories();
+ public IBinaryTokenizerFactory getTokenizerFactory();
+
public ITypeTraits[] getInvListsTypeTraits();
public IFileSplitProvider getInvListsFileSplitProvider();
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearchModifier.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearchModifier.java
index 8f7a3e3..bd96f67 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearchModifier.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearchModifier.java
@@ -18,7 +18,7 @@
import java.util.List;
public interface IInvertedIndexSearchModifier {
- int getOccurrenceThreshold(List<IInvertedListCursor> invListCursors);
+ public int getOccurrenceThreshold(List<IInvertedListCursor> invListCursors);
- int getPrefixLists(List<IInvertedListCursor> invListCursors);
+ public int getPrefixLists(List<IInvertedListCursor> invListCursors);
}
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearcher.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearcher.java
index 36d64ea..d633c34 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearcher.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearcher.java
@@ -19,11 +19,15 @@
import java.util.List;
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndexSearchCursor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndexSearchPredicate;
public interface IInvertedIndexSearcher {
- public void search(IInvertedIndexResultCursor resultCursor, ITupleReference queryTuple, int queryFieldIndex,
- IInvertedIndexSearchModifier searchModifier) throws Exception;
+ public void search(InvertedIndexSearchCursor resultCursor, InvertedIndexSearchPredicate searchPred)
+ throws HyracksDataException, IndexException;
public IFrameTupleAccessor createResultFrameTupleAccessor();
@@ -32,4 +36,6 @@
public List<ByteBuffer> getResultBuffers();
public int getNumValidResultBuffers();
+
+ public void reset();
}
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/AbstractInvertedIndexOperatorDescriptor.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/AbstractInvertedIndexOperatorDescriptor.java
index eab266d..117f96f 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/AbstractInvertedIndexOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/AbstractInvertedIndexOperatorDescriptor.java
@@ -18,18 +18,21 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITupleFilterFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizerFactory;
import edu.uci.ics.hyracks.storage.am.invertedindex.util.InvertedIndexUtils;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
@@ -41,7 +44,9 @@
// General.
protected final IStorageManagerInterface storageManager;
protected final IIndexRegistryProvider<IIndex> indexRegistryProvider;
-
+ protected final boolean retainInput;
+ protected final IOperationCallbackProvider opCallbackProvider;
+
// Btree.
protected final ITreeIndexFrameFactory btreeInteriorFrameFactory;
protected final ITreeIndexFrameFactory btreeLeafFrameFactory;
@@ -53,21 +58,24 @@
// Inverted index.
protected final ITypeTraits[] invListsTypeTraits;
protected final IBinaryComparatorFactory[] invListComparatorFactories;
+ protected final IBinaryTokenizerFactory tokenizerFactory;
protected final IFileSplitProvider invListsFileSplitProvider;
- public AbstractInvertedIndexOperatorDescriptor(JobSpecification spec, int inputArity, int outputArity,
+ public AbstractInvertedIndexOperatorDescriptor(IOperatorDescriptorRegistry spec, int inputArity, int outputArity,
RecordDescriptor recDesc, IStorageManagerInterface storageManager,
IFileSplitProvider btreeFileSplitProvider, IFileSplitProvider invListsFileSplitProvider,
IIndexRegistryProvider<IIndex> indexRegistryProvider, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenComparatorFactories, ITypeTraits[] invListsTypeTraits,
- IBinaryComparatorFactory[] invListComparatorFactories,
- IIndexDataflowHelperFactory btreeDataflowHelperFactory) {
+ IBinaryComparatorFactory[] invListComparatorFactories, IBinaryTokenizerFactory tokenizerFactory,
+ IIndexDataflowHelperFactory btreeDataflowHelperFactory, boolean retainInput, IOperationCallbackProvider opCallbackProvider) {
super(spec, inputArity, outputArity);
// General.
this.storageManager = storageManager;
this.indexRegistryProvider = indexRegistryProvider;
-
+ this.retainInput = retainInput;
+ this.opCallbackProvider = opCallbackProvider;
+
// Btree.
this.btreeTypeTraits = InvertedIndexUtils.getBTreeTypeTraits(tokenTypeTraits);
ITreeIndexTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(btreeTypeTraits);
@@ -80,6 +88,7 @@
// Inverted index.
this.invListsTypeTraits = invListsTypeTraits;
this.invListComparatorFactories = invListComparatorFactories;
+ this.tokenizerFactory = tokenizerFactory;
this.invListsFileSplitProvider = invListsFileSplitProvider;
if (outputArity > 0) {
@@ -108,16 +117,6 @@
}
@Override
- public ITreeIndexFrameFactory getTreeIndexInteriorFactory() {
- return btreeInteriorFrameFactory;
- }
-
- @Override
- public ITreeIndexFrameFactory getTreeIndexLeafFactory() {
- return btreeLeafFrameFactory;
- }
-
- @Override
public IStorageManagerInterface getStorageManager() {
return storageManager;
}
@@ -133,6 +132,11 @@
}
@Override
+ public IBinaryTokenizerFactory getTokenizerFactory() {
+ return tokenizerFactory;
+ }
+
+ @Override
public ITypeTraits[] getInvListsTypeTraits() {
return invListsTypeTraits;
}
@@ -146,4 +150,19 @@
public IIndexDataflowHelperFactory getIndexDataflowHelperFactory() {
return btreeDataflowHelperFactory;
}
+
+ @Override
+ public boolean getRetainInput() {
+ return retainInput;
+ }
+
+ @Override
+ public IOperationCallbackProvider getOpCallbackProvider() {
+ return opCallbackProvider;
+ }
+
+ @Override
+ public ITupleFilterFactory getTupleFilterFactory() {
+ return null;
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/BinaryTokenizerOperatorDescriptor.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/BinaryTokenizerOperatorDescriptor.java
index ffb94e7..a146479 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/BinaryTokenizerOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/BinaryTokenizerOperatorDescriptor.java
@@ -20,7 +20,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizerFactory;
@@ -36,7 +36,7 @@
// WARNING: too many key fields can cause significant data blowup.
private final int[] keyFields;
- public BinaryTokenizerOperatorDescriptor(JobSpecification spec, RecordDescriptor recDesc,
+ public BinaryTokenizerOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc,
IBinaryTokenizerFactory tokenizerFactory, int[] tokenFields, int[] keyFields) {
super(spec, 1, 1);
this.tokenizerFactory = tokenizerFactory;
@@ -46,9 +46,9 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider,
- int partition, int nPartitions) throws HyracksDataException {
- return new BinaryTokenizerOperatorNodePushable(ctx, recordDescProvider.getInputRecordDescriptor(odId, 0),
- recordDescriptors[0], tokenizerFactory.createTokenizer(), tokenFields, keyFields);
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
+ return new BinaryTokenizerOperatorNodePushable(ctx, recordDescProvider.getInputRecordDescriptor(
+ getActivityId(), 0), recordDescriptors[0], tokenizerFactory.createTokenizer(), tokenFields, keyFields);
}
}
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorDescriptor.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorDescriptor.java
index e91aa08..7ffec4d 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorDescriptor.java
@@ -20,11 +20,13 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
+import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizerFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
public class InvertedIndexBulkLoadOperatorDescriptor extends AbstractInvertedIndexOperatorDescriptor {
@@ -33,15 +35,17 @@
private final int[] fieldPermutation;
- public InvertedIndexBulkLoadOperatorDescriptor(JobSpecification spec, int[] fieldPermutation,
+ public InvertedIndexBulkLoadOperatorDescriptor(IOperatorDescriptorRegistry spec, int[] fieldPermutation,
IStorageManagerInterface storageManager, IFileSplitProvider btreeFileSplitProvider,
IFileSplitProvider invListsFileSplitProvider, IIndexRegistryProvider<IIndex> indexRegistryProvider,
ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenComparatorFactories,
ITypeTraits[] invListsTypeTraits, IBinaryComparatorFactory[] invListComparatorFactories,
- IIndexDataflowHelperFactory btreeDataflowHelperFactory) {
+ IBinaryTokenizerFactory tokenizerFactory, IIndexDataflowHelperFactory btreeDataflowHelperFactory,
+ IOperationCallbackProvider opCallbackProvider) {
super(spec, 1, 0, null, storageManager, btreeFileSplitProvider, invListsFileSplitProvider,
indexRegistryProvider, tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits,
- invListComparatorFactories, btreeDataflowHelperFactory);
+ invListComparatorFactories, tokenizerFactory, btreeDataflowHelperFactory, false,
+ opCallbackProvider);
this.fieldPermutation = fieldPermutation;
}
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java
index 9dcabb4..c6fa56d 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java
@@ -23,21 +23,16 @@
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
import edu.uci.ics.hyracks.storage.am.common.dataflow.PermutingFrameTupleReference;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
-import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
-import edu.uci.ics.hyracks.storage.am.invertedindex.impls.FixedSizeElementInvertedListBuilder;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex.InvertedIndexBulkLoadContext;
public class InvertedIndexBulkLoadOperatorNodePushable extends AbstractUnaryInputSinkOperatorNodePushable {
private final TreeIndexDataflowHelper btreeDataflowHelper;
private final InvertedIndexDataflowHelper invIndexDataflowHelper;
- private final IInvertedListBuilder invListBuilder;
private InvertedIndex invIndex;
- private InvertedIndex.BulkLoadContext bulkLoadCtx;
-
- private final IHyracksTaskContext ctx;
+ private InvertedIndex.InvertedIndexBulkLoadContext bulkLoadCtx;
private FrameTupleAccessor accessor;
private PermutingFrameTupleReference tuple = new PermutingFrameTupleReference();
@@ -47,11 +42,9 @@
public InvertedIndexBulkLoadOperatorNodePushable(AbstractInvertedIndexOperatorDescriptor opDesc,
IHyracksTaskContext ctx, int partition, int[] fieldPermutation, IRecordDescriptorProvider recordDescProvider) {
btreeDataflowHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory()
- .createIndexDataflowHelper(opDesc, ctx, partition, true);
- invIndexDataflowHelper = new InvertedIndexDataflowHelper(btreeDataflowHelper, opDesc, ctx, partition, true);
+ .createIndexDataflowHelper(opDesc, ctx, partition);
+ invIndexDataflowHelper = new InvertedIndexDataflowHelper(btreeDataflowHelper, opDesc, ctx, partition);
this.recordDescProvider = recordDescProvider;
- this.ctx = ctx;
- this.invListBuilder = new FixedSizeElementInvertedListBuilder(opDesc.getInvListsTypeTraits());
tuple.setFieldPermutation(fieldPermutation);
}
@@ -59,12 +52,12 @@
public void open() throws HyracksDataException {
AbstractInvertedIndexOperatorDescriptor opDesc = (AbstractInvertedIndexOperatorDescriptor) btreeDataflowHelper
.getOperatorDescriptor();
- RecordDescriptor recDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getOperatorId(), 0);
+ RecordDescriptor recDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getActivityId(), 0);
accessor = new FrameTupleAccessor(btreeDataflowHelper.getHyracksTaskContext().getFrameSize(), recDesc);
// BTree.
try {
- btreeDataflowHelper.init();
+ btreeDataflowHelper.init(false);
} catch (Exception e) {
// Cleanup in case of failure.
btreeDataflowHelper.deinit();
@@ -77,9 +70,9 @@
// Inverted Index.
try {
- invIndexDataflowHelper.init();
+ invIndexDataflowHelper.init(false);
invIndex = (InvertedIndex) invIndexDataflowHelper.getIndex();
- bulkLoadCtx = invIndex.beginBulkLoad(invListBuilder, ctx.getFrameSize(), BTree.DEFAULT_FILL_FACTOR);
+ bulkLoadCtx = (InvertedIndexBulkLoadContext) invIndex.beginBulkLoad(BTree.DEFAULT_FILL_FACTOR);
} catch (Exception e) {
// Cleanup in case of failure.
invIndexDataflowHelper.deinit();
@@ -97,11 +90,7 @@
int tupleCount = accessor.getTupleCount();
for (int i = 0; i < tupleCount; i++) {
tuple.reset(accessor, i);
- try {
- invIndex.bulkLoadAddTuple(bulkLoadCtx, tuple);
- } catch (PageAllocationException e) {
- throw new HyracksDataException(e);
- }
+ invIndex.bulkLoadAddTuple(tuple, bulkLoadCtx);
}
}
@@ -109,7 +98,7 @@
public void close() throws HyracksDataException {
try {
invIndex.endBulkLoad(bulkLoadCtx);
- } catch (PageAllocationException e) {
+ } catch (Exception e) {
throw new HyracksDataException(e);
} finally {
try {
@@ -122,5 +111,6 @@
@Override
public void fail() throws HyracksDataException {
+ writer.fail();
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexCreateOperatorDescriptor.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexCreateOperatorDescriptor.java
new file mode 100644
index 0000000..01d1a97
--- /dev/null
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexCreateOperatorDescriptor.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.invertedindex.dataflow;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
+import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizerFactory;
+import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+
+public class InvertedIndexCreateOperatorDescriptor extends AbstractInvertedIndexOperatorDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ public InvertedIndexCreateOperatorDescriptor(IOperatorDescriptorRegistry spec,
+ IStorageManagerInterface storageManager, IFileSplitProvider btreeFileSplitProvider,
+ IFileSplitProvider invListsFileSplitProvider, IIndexRegistryProvider<IIndex> indexRegistryProvider,
+ ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenComparatorFactories,
+ ITypeTraits[] invListsTypeTraits, IBinaryComparatorFactory[] invListComparatorFactories,
+ IBinaryTokenizerFactory tokenizerFactory, IIndexDataflowHelperFactory btreeDataflowHelperFactory,
+ IOperationCallbackProvider opCallbackProvider) {
+ super(spec, 0, 0, null, storageManager, btreeFileSplitProvider, invListsFileSplitProvider,
+ indexRegistryProvider, tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits,
+ invListComparatorFactories, tokenizerFactory, btreeDataflowHelperFactory, false, opCallbackProvider);
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ return new InvertedIndexCreateOperatorNodePushable(this, ctx, partition);
+ }
+}
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexCreateOperatorNodePushable.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexCreateOperatorNodePushable.java
new file mode 100644
index 0000000..eb909d8
--- /dev/null
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexCreateOperatorNodePushable.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.invertedindex.dataflow;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorNodePushable;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
+
+public class InvertedIndexCreateOperatorNodePushable extends AbstractOperatorNodePushable {
+ private final TreeIndexDataflowHelper btreeDataflowHelper;
+ private final InvertedIndexDataflowHelper invIndexDataflowHelper;
+
+ public InvertedIndexCreateOperatorNodePushable(AbstractInvertedIndexOperatorDescriptor opDesc,
+ IHyracksTaskContext ctx, int partition) {
+ btreeDataflowHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory()
+ .createIndexDataflowHelper(opDesc, ctx, partition);
+ invIndexDataflowHelper = new InvertedIndexDataflowHelper(btreeDataflowHelper, opDesc, ctx, partition);
+ }
+
+ @Override
+ public void deinitialize() throws HyracksDataException {
+ }
+
+ @Override
+ public int getInputArity() {
+ return 0;
+ }
+
+ @Override
+ public IFrameWriter getInputFrameWriter(int index) {
+ return null;
+ }
+
+ @Override
+ public void initialize() throws HyracksDataException {
+ // BTree.
+ try {
+ btreeDataflowHelper.init(true);
+ } finally {
+ btreeDataflowHelper.deinit();
+ }
+ // Inverted Index.
+ try {
+ invIndexDataflowHelper.init(true);
+ } finally {
+ invIndexDataflowHelper.deinit();
+ }
+ }
+
+ @Override
+ public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) {
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexDataflowHelper.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexDataflowHelper.java
index 71717e4..6d0ff38 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexDataflowHelper.java
@@ -23,17 +23,17 @@
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.common.util.IndexUtils;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.FixedSizeElementInvertedListBuilder;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
public final class InvertedIndexDataflowHelper extends IndexDataflowHelper {
private final TreeIndexDataflowHelper btreeDataflowHelper;
public InvertedIndexDataflowHelper(TreeIndexDataflowHelper btreeDataflowHelper, IIndexOperatorDescriptor opDesc,
- IHyracksTaskContext ctx, int partition, boolean createIfNotExists) {
- super(opDesc, ctx, partition, createIfNotExists);
+ IHyracksTaskContext ctx, int partition) {
+ super(opDesc, ctx, partition);
this.btreeDataflowHelper = btreeDataflowHelper;
}
@@ -46,10 +46,12 @@
@Override
public IIndex createIndexInstance() throws HyracksDataException {
IInvertedIndexOperatorDescriptor invIndexOpDesc = (IInvertedIndexOperatorDescriptor) opDesc;
- MultiComparator cmp = IndexUtils.createMultiComparator(invIndexOpDesc.getInvListsComparatorFactories());
// Assumes btreeDataflowHelper.init() has already been called.
BTree btree = (BTree) btreeDataflowHelper.getIndex();
+ IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(
+ invIndexOpDesc.getInvListsTypeTraits());
return new InvertedIndex(opDesc.getStorageManager().getBufferCache(ctx), btree,
- invIndexOpDesc.getInvListsTypeTraits(), cmp);
+ invIndexOpDesc.getInvListsTypeTraits(), invIndexOpDesc.getInvListsComparatorFactories(),
+ invListBuilder);
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexSearchOperatorDescriptor.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexSearchOperatorDescriptor.java
index bf9899f..497dfb5 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexSearchOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexSearchOperatorDescriptor.java
@@ -22,14 +22,14 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifierFactory;
-import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizer;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizerFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
@@ -37,30 +37,29 @@
private static final long serialVersionUID = 1L;
private final int queryField;
- private final IBinaryTokenizerFactory queryTokenizerFactory;
private final IInvertedIndexSearchModifierFactory searchModifierFactory;
- public InvertedIndexSearchOperatorDescriptor(JobSpecification spec,
- int queryField, IStorageManagerInterface storageManager, IFileSplitProvider btreeFileSplitProvider,
+ public InvertedIndexSearchOperatorDescriptor(IOperatorDescriptorRegistry spec, int queryField,
+ IStorageManagerInterface storageManager, IFileSplitProvider btreeFileSplitProvider,
IFileSplitProvider invListsFileSplitProvider, IIndexRegistryProvider<IIndex> indexRegistryProvider,
ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenComparatorFactories,
ITypeTraits[] invListsTypeTraits, IBinaryComparatorFactory[] invListComparatorFactories,
IIndexDataflowHelperFactory btreeDataflowHelperFactory, IBinaryTokenizerFactory queryTokenizerFactory,
- IInvertedIndexSearchModifierFactory searchModifierFactory, RecordDescriptor recDesc) {
+ IInvertedIndexSearchModifierFactory searchModifierFactory, RecordDescriptor recDesc, boolean retainInput,
+ IOperationCallbackProvider opCallbackProvider) {
super(spec, 1, 1, recDesc, storageManager, btreeFileSplitProvider, invListsFileSplitProvider,
indexRegistryProvider, tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits,
- invListComparatorFactories, btreeDataflowHelperFactory);
+ invListComparatorFactories, queryTokenizerFactory, btreeDataflowHelperFactory, retainInput,
+ opCallbackProvider);
this.queryField = queryField;
- this.queryTokenizerFactory = queryTokenizerFactory;
this.searchModifierFactory = searchModifierFactory;
}
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
- IBinaryTokenizer tokenizer = queryTokenizerFactory.createTokenizer();
IInvertedIndexSearchModifier searchModifier = searchModifierFactory.createSearchModifier();
- return new InvertedIndexSearchOperatorNodePushable(this, ctx, partition, queryField, searchModifier, tokenizer,
+ return new InvertedIndexSearchOperatorNodePushable(this, ctx, partition, queryField, searchModifier,
recordDescProvider);
}
}
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexSearchOperatorNodePushable.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexSearchOperatorNodePushable.java
index ceaff1e..f3080f4 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexSearchOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexSearchOperatorNodePushable.java
@@ -29,29 +29,26 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
-import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexResultCursor;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndexSearchPredicate;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.OccurrenceThresholdPanicException;
-import edu.uci.ics.hyracks.storage.am.invertedindex.impls.SearchResultCursor;
-import edu.uci.ics.hyracks.storage.am.invertedindex.impls.TOccurrenceSearcher;
-import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizer;
public class InvertedIndexSearchOperatorNodePushable extends AbstractUnaryInputUnaryOutputOperatorNodePushable {
private final TreeIndexDataflowHelper btreeDataflowHelper;
private final InvertedIndexDataflowHelper invIndexDataflowHelper;
- private final IHyracksTaskContext ctx;
private final int queryField;
private FrameTupleAccessor accessor;
private FrameTupleReference tuple;
private IRecordDescriptorProvider recordDescProvider;
private InvertedIndex invIndex;
- private final IInvertedIndexSearchModifier searchModifier;
- private final IBinaryTokenizer queryTokenizer;
- private TOccurrenceSearcher searcher;
- private IInvertedIndexResultCursor resultCursor;
+ private final InvertedIndexSearchPredicate searchPred;
+ private IIndexAccessor indexAccessor;
+ private IIndexCursor resultCursor;
private ByteBuffer writeBuffer;
private FrameTupleAppender appender;
@@ -59,29 +56,29 @@
private DataOutput dos;
private final AbstractInvertedIndexOperatorDescriptor opDesc;
+ private final boolean retainInput;
public InvertedIndexSearchOperatorNodePushable(AbstractInvertedIndexOperatorDescriptor opDesc,
IHyracksTaskContext ctx, int partition, int queryField, IInvertedIndexSearchModifier searchModifier,
- IBinaryTokenizer queryTokenizer, IRecordDescriptorProvider recordDescProvider) {
+ IRecordDescriptorProvider recordDescProvider) {
this.opDesc = opDesc;
btreeDataflowHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory()
- .createIndexDataflowHelper(opDesc, ctx, partition, false);
- invIndexDataflowHelper = new InvertedIndexDataflowHelper(btreeDataflowHelper, opDesc, ctx, partition, false);
- this.ctx = ctx;
+ .createIndexDataflowHelper(opDesc, ctx, partition);
+ invIndexDataflowHelper = new InvertedIndexDataflowHelper(btreeDataflowHelper, opDesc, ctx, partition);
this.queryField = queryField;
- this.searchModifier = searchModifier;
- this.queryTokenizer = queryTokenizer;
+ this.searchPred = new InvertedIndexSearchPredicate(opDesc.getTokenizerFactory().createTokenizer(), searchModifier);
this.recordDescProvider = recordDescProvider;
+ this.retainInput = invIndexDataflowHelper.getOperatorDescriptor().getRetainInput();
}
@Override
public void open() throws HyracksDataException {
- RecordDescriptor recDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getOperatorId(), 0);
- accessor = new FrameTupleAccessor(btreeDataflowHelper.getHyracksTaskContext().getFrameSize(), recDesc);
+ RecordDescriptor inputRecDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getActivityId(), 0);
+ accessor = new FrameTupleAccessor(btreeDataflowHelper.getHyracksTaskContext().getFrameSize(), inputRecDesc);
tuple = new FrameTupleReference();
// BTree.
try {
- btreeDataflowHelper.init();
+ btreeDataflowHelper.init(false);
} catch (Exception e) {
// Cleanup in case of failure/
btreeDataflowHelper.deinit();
@@ -93,7 +90,7 @@
}
// Inverted Index.
try {
- invIndexDataflowHelper.init();
+ invIndexDataflowHelper.init(false);
invIndex = (InvertedIndex) invIndexDataflowHelper.getIndex();
} catch (Exception e) {
// Cleanup in case of failure.
@@ -106,15 +103,14 @@
}
writeBuffer = btreeDataflowHelper.getHyracksTaskContext().allocateFrame();
- tb = new ArrayTupleBuilder(opDesc.getInvListsTypeTraits().length);
+ tb = new ArrayTupleBuilder(recordDesc.getFieldCount());
dos = tb.getDataOutput();
appender = new FrameTupleAppender(btreeDataflowHelper.getHyracksTaskContext().getFrameSize());
appender.reset(writeBuffer, true);
- searcher = new TOccurrenceSearcher(ctx, invIndex, queryTokenizer);
- resultCursor = new SearchResultCursor(searcher.createResultFrameTupleAccessor(),
- searcher.createResultTupleReference());
-
+ indexAccessor = invIndex.createAccessor();
+ //InvertedIndex.InvertedIndexAccessor accessor =
+ resultCursor = indexAccessor.createSearchCursor();
writer.open();
}
@@ -122,6 +118,12 @@
while (resultCursor.hasNext()) {
resultCursor.next();
tb.reset();
+ if (retainInput) {
+ for (int i = 0; i < tuple.getFieldCount(); i++) {
+ dos.write(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
+ tb.addFieldEndOffset();
+ }
+ }
ITupleReference invListElement = resultCursor.getTuple();
int invListFields = opDesc.getInvListsTypeTraits().length;
for (int i = 0; i < invListFields; i++) {
@@ -146,10 +148,11 @@
try {
for (int i = 0; i < tupleCount; i++) {
tuple.reset(accessor, i);
- searcher.reset();
+ searchPred.setQueryTuple(tuple);
+ searchPred.setQueryFieldIndex(queryField);
try {
- searcher.reset();
- searcher.search(resultCursor, tuple, queryField, searchModifier);
+ resultCursor.reset();
+ indexAccessor.search(resultCursor, searchPred);
writeSearchResults();
} catch (OccurrenceThresholdPanicException e) {
// Ignore panic cases for now.
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListBuilder.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListBuilder.java
index 643c105..c4ab601 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListBuilder.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListBuilder.java
@@ -36,7 +36,7 @@
@Override
public boolean startNewList(ITupleReference tuple, int tokenField) {
- if (pos + listElementSize > targetBuf.length)
+ if (pos + listElementSize >= targetBuf.length)
return false;
else {
listSize = 0;
@@ -46,7 +46,7 @@
@Override
public boolean appendElement(ITupleReference tuple, int numTokenFields, int numElementFields) {
- if (pos + listElementSize > targetBuf.length)
+ if (pos + listElementSize >= targetBuf.length)
return false;
for (int i = 0; i < numElementFields; i++) {
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListCursor.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListCursor.java
index 4158019..446f171 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListCursor.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListCursor.java
@@ -57,7 +57,7 @@
@Override
public void next() {
- if (currentOff + elementSize >= bufferCache.getPageSize()) {
+ if (currentOff + 2 * elementSize >= bufferCache.getPageSize()) {
currentPageIx++;
currentOff = 0;
} else {
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
index 986e57b..99e76dc 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
@@ -17,26 +17,29 @@
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.context.IHyracksCommonContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ByteArrayAccessibleOutputStream;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
-import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearcher;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListCursor;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
@@ -45,30 +48,33 @@
/**
* An inverted index consists of two files: 1. a file storing (paginated)
- * inverted lists 2. a BTree-file mapping from tokens to inverted lists
- *
+ * inverted lists 2. a BTree-file mapping from tokens to inverted lists.
* Implemented features: bulk loading and searching (based on T-Occurrence) Not
* implemented features: updates (insert/update/delete) Limitations: a query
- * cannot exceed the size of a Hyracks frame
+ * cannot exceed the size of a Hyracks frame.
*/
public class InvertedIndex implements IIndex {
+ private final IHyracksCommonContext ctx = new DefaultHyracksCommonContext();
private BTree btree;
private int rootPageId = 0;
private IBufferCache bufferCache;
private int fileId;
private final ITypeTraits[] invListTypeTraits;
- private final MultiComparator invListCmp;
+ private final IBinaryComparatorFactory[] invListCmpFactories;
+ private final IInvertedListBuilder invListBuilder;
private final int numTokenFields;
private final int numInvListKeys;
- public InvertedIndex(IBufferCache bufferCache, BTree btree, ITypeTraits[] invListTypeTraits, MultiComparator invListCmp) {
+ public InvertedIndex(IBufferCache bufferCache, BTree btree, ITypeTraits[] invListTypeTraits,
+ IBinaryComparatorFactory[] invListCmpFactories, IInvertedListBuilder invListBuilder) {
this.bufferCache = bufferCache;
this.btree = btree;
- this.invListCmp = invListCmp;
this.invListTypeTraits = invListTypeTraits;
- this.numTokenFields = btree.getMultiComparator().getKeyFieldCount();
- this.numInvListKeys = invListCmp.getKeyFieldCount();
+ this.invListCmpFactories = invListCmpFactories;
+ this.invListBuilder = invListBuilder;
+ this.numTokenFields = btree.getComparatorFactories().length;
+ this.numInvListKeys = invListCmpFactories.length;
}
@Override
@@ -79,208 +85,160 @@
@Override
public void create(int indexFileId) throws HyracksDataException {
}
-
+
+ @Override
public void close() {
this.fileId = -1;
}
- public BulkLoadContext beginBulkLoad(IInvertedListBuilder invListBuilder, int hyracksFrameSize,
- float btreeFillFactor) throws HyracksDataException, TreeIndexException, PageAllocationException {
- BulkLoadContext ctx = new BulkLoadContext(invListBuilder, hyracksFrameSize, btreeFillFactor);
+ public boolean openCursor(ITreeIndexCursor btreeCursor, RangePredicate btreePred, ITreeIndexAccessor btreeAccessor,
+ IInvertedListCursor invListCursor) throws HyracksDataException, IndexException {
+ btreeAccessor.search(btreeCursor, btreePred);
+ boolean ret = false;
+ try {
+ if (btreeCursor.hasNext()) {
+ btreeCursor.next();
+ ITupleReference frameTuple = btreeCursor.getTuple();
+ // Hardcoded mapping of btree fields
+ int startPageId = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(1),
+ frameTuple.getFieldStart(1));
+ int endPageId = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(2),
+ frameTuple.getFieldStart(2));
+ int startOff = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(3),
+ frameTuple.getFieldStart(3));
+ int numElements = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(4),
+ frameTuple.getFieldStart(4));
+ invListCursor.reset(startPageId, endPageId, startOff, numElements);
+ ret = true;
+ } else {
+ invListCursor.reset(0, 0, 0, 0);
+ }
+ } finally {
+ btreeCursor.close();
+ btreeCursor.reset();
+ }
+ return ret;
+ }
+
+ @Override
+ public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws TreeIndexException, HyracksDataException {
+ InvertedIndexBulkLoadContext ctx = new InvertedIndexBulkLoadContext(fillFactor);
ctx.init(rootPageId, fileId);
return ctx;
}
- // ASSUMPTIONS:
- // the first btree.getMultiComparator().getKeyFieldCount() fields in tuple
- // are btree keys (e.g., a string token)
- // the next invListCmp.getKeyFieldCount() fields in tuple are keys of the
- // inverted list (e.g., primary key)
- // key fields of inverted list are fixed size
- public void bulkLoadAddTuple(BulkLoadContext ctx, ITupleReference tuple) throws HyracksDataException, PageAllocationException {
-
- // first inverted list, copy token to baaos and start new list
- if (ctx.currentInvListTokenBaaos.size() == 0) {
- ctx.currentInvListStartPageId = ctx.currentPageId;
- ctx.currentInvListStartOffset = ctx.invListBuilder.getPos();
-
- // remember current token
- ctx.currentInvListTokenBaaos.reset();
- for (int i = 0; i < numTokenFields; i++) {
- ctx.currentInvListTokenBaaos.write(tuple.getFieldData(i), tuple.getFieldStart(i),
- tuple.getFieldLength(i));
+ /**
+ * Assumptions:
+ * The first btree.getMultiComparator().getKeyFieldCount() fields in tuple
+ * are btree keys (e.g., a string token).
+ * The next invListCmp.getKeyFieldCount() fields in tuple are keys of the
+ * inverted list (e.g., primary key).
+ * Key fields of inverted list are fixed size.
+ */
+ @Override
+ public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException {
+ InvertedIndexBulkLoadContext ctx = (InvertedIndexBulkLoadContext) ictx;
+ boolean firstElement = ctx.lastTupleBuilder.getSize() == 0;
+ boolean startNewList = firstElement;
+ if (!firstElement) {
+ // If the current and the last token don't match, we start a new list.
+ ctx.lastTuple.reset(ctx.lastTupleBuilder.getFieldEndOffsets(), ctx.lastTupleBuilder.getByteArray());
+ startNewList = ctx.tokenCmp.compare(tuple, ctx.lastTuple) != 0;
+ }
+ if (startNewList) {
+ if (!firstElement) {
+ // Create entry in btree for last inverted list.
+ createAndInsertBTreeTuple(ctx);
}
-
- if (!ctx.invListBuilder.startNewList(tuple, numTokenFields)) {
+ if (!invListBuilder.startNewList(tuple, numTokenFields)) {
ctx.pinNextPage();
- ctx.invListBuilder.setTargetBuffer(ctx.currentPage.getBuffer().array(), 0);
- if (!ctx.invListBuilder.startNewList(tuple, numTokenFields)) {
+ invListBuilder.setTargetBuffer(ctx.currentPage.getBuffer().array(), 0);
+ if (!invListBuilder.startNewList(tuple, numTokenFields)) {
throw new IllegalStateException("Failed to create first inverted list.");
}
}
- }
-
- // create new inverted list?
- ctx.currentInvListToken.reset(ctx.currentInvListTokenBaaos.getByteArray(), 0);
- if (ctx.tokenCmp.compare(tuple, ctx.currentInvListToken) != 0) {
-
- // create entry in btree for last inverted list
- createAndInsertBTreeTuple(ctx);
-
- // remember new token
- ctx.currentInvListTokenBaaos.reset();
- for (int i = 0; i < numTokenFields; i++) {
- ctx.currentInvListTokenBaaos.write(tuple.getFieldData(i), tuple.getFieldStart(i),
- tuple.getFieldLength(i));
- }
-
- // start new list
- if (!ctx.invListBuilder.startNewList(tuple, numTokenFields)) {
- ctx.pinNextPage();
- ctx.invListBuilder.setTargetBuffer(ctx.currentPage.getBuffer().array(), 0);
- if (!ctx.invListBuilder.startNewList(tuple, numTokenFields)) {
- throw new IllegalStateException("Failed to start new inverted list after switching to a new page.");
- }
- }
-
ctx.currentInvListStartPageId = ctx.currentPageId;
- ctx.currentInvListStartOffset = ctx.invListBuilder.getPos();
+ ctx.currentInvListStartOffset = invListBuilder.getPos();
+ } else {
+ if (ctx.invListCmp.compare(tuple, ctx.lastTuple, numTokenFields) == 0) {
+ // Duplicate inverted-list element.
+ return;
+ }
}
- // append to current inverted list
- if (!ctx.invListBuilder.appendElement(tuple, numTokenFields, numInvListKeys)) {
+ // Append to current inverted list.
+ if (!invListBuilder.appendElement(tuple, numTokenFields, numInvListKeys)) {
ctx.pinNextPage();
- ctx.invListBuilder.setTargetBuffer(ctx.currentPage.getBuffer().array(), 0);
- if (!ctx.invListBuilder.appendElement(tuple, numTokenFields, numInvListKeys)) {
+ invListBuilder.setTargetBuffer(ctx.currentPage.getBuffer().array(), 0);
+ if (!invListBuilder.appendElement(tuple, numTokenFields, numInvListKeys)) {
throw new IllegalStateException(
"Failed to append element to inverted list after switching to a new page.");
}
}
- }
- public boolean openCursor(ITreeIndexCursor btreeCursor, RangePredicate btreePred, ITreeIndexAccessor btreeAccessor,
- IInvertedListCursor invListCursor) throws Exception {
- btreeAccessor.search(btreeCursor, btreePred);
-
- boolean ret = false;
- if (btreeCursor.hasNext()) {
-
- btreeCursor.next();
- ITupleReference frameTuple = btreeCursor.getTuple();
-
- // hardcoded mapping of btree fields
- int startPageId = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(1),
- frameTuple.getFieldStart(1));
- int endPageId = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(2),
- frameTuple.getFieldStart(2));
- int startOff = IntegerSerializerDeserializer
- .getInt(frameTuple.getFieldData(3), frameTuple.getFieldStart(3));
- int numElements = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(4),
- frameTuple.getFieldStart(4));
-
- invListCursor.reset(startPageId, endPageId, startOff, numElements);
- ret = true;
- } else {
- invListCursor.reset(0, 0, 0, 0);
+ // Remember last tuple by creating a copy.
+ // TODO: This portion can be optimized by only copying the token when it changes, and using the last appended inverted-list element as a reference.
+ ctx.lastTupleBuilder.reset();
+ for (int i = 0; i < tuple.getFieldCount(); i++) {
+ ctx.lastTupleBuilder.addField(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
}
-
- btreeCursor.close();
- btreeCursor.reset();
-
- return ret;
}
- public void createAndInsertBTreeTuple(BulkLoadContext ctx) throws HyracksDataException, PageAllocationException {
- // build tuple
+ private void createAndInsertBTreeTuple(InvertedIndexBulkLoadContext ctx) throws HyracksDataException {
+ // Build tuple.
ctx.btreeTupleBuilder.reset();
- ctx.btreeTupleBuilder.addField(ctx.currentInvListTokenBaaos.getByteArray(), 0,
- ctx.currentInvListTokenBaaos.size());
+ ctx.btreeTupleBuilder.addField(ctx.lastTuple.getFieldData(0), ctx.lastTuple.getFieldStart(0),
+ ctx.lastTuple.getFieldLength(0));
ctx.btreeTupleBuilder.addField(IntegerSerializerDeserializer.INSTANCE, ctx.currentInvListStartPageId);
ctx.btreeTupleBuilder.addField(IntegerSerializerDeserializer.INSTANCE, ctx.currentPageId);
ctx.btreeTupleBuilder.addField(IntegerSerializerDeserializer.INSTANCE, ctx.currentInvListStartOffset);
- ctx.btreeTupleBuilder.addField(IntegerSerializerDeserializer.INSTANCE, ctx.invListBuilder.getListSize());
-
- // append to buffer
- ctx.btreeTupleAppender.reset(ctx.btreeTupleBuffer, true);
- ctx.btreeTupleAppender.append(ctx.btreeTupleBuilder.getFieldEndOffsets(), ctx.btreeTupleBuilder.getByteArray(),
- 0, ctx.btreeTupleBuilder.getSize());
-
- // reset tuple reference
- ctx.btreeFrameTupleReference.reset(ctx.btreeFrameTupleAccessor, 0);
-
- btree.bulkLoadAddTuple(ctx.btreeFrameTupleReference, ctx.btreeBulkLoadCtx);
+ ctx.btreeTupleBuilder.addField(IntegerSerializerDeserializer.INSTANCE, invListBuilder.getListSize());
+ // Reset tuple reference and add it.
+ ctx.btreeTupleReference.reset(ctx.btreeTupleBuilder.getFieldEndOffsets(), ctx.btreeTupleBuilder.getByteArray());
+ btree.bulkLoadAddTuple(ctx.btreeTupleReference, ctx.btreeBulkLoadCtx);
}
- public void endBulkLoad(BulkLoadContext ctx) throws HyracksDataException, PageAllocationException {
- // create entry in btree for last inverted list
+ @Override
+ public void endBulkLoad(IIndexBulkLoadContext ictx) throws HyracksDataException {
+ // Create entry in btree for last inverted list.
+ InvertedIndexBulkLoadContext ctx = (InvertedIndexBulkLoadContext) ictx;
createAndInsertBTreeTuple(ctx);
btree.endBulkLoad(ctx.btreeBulkLoadCtx);
ctx.deinit();
}
- public IBufferCache getBufferCache() {
- return bufferCache;
- }
-
- public int getInvListsFileId() {
- return fileId;
- }
-
- public MultiComparator getInvListElementCmp() {
- return invListCmp;
- }
-
- public ITypeTraits[] getTypeTraits() {
- return invListTypeTraits;
- }
-
- public BTree getBTree() {
- return btree;
- }
-
- public final class BulkLoadContext {
- private final ByteBuffer btreeTupleBuffer;
+ public final class InvertedIndexBulkLoadContext implements IIndexBulkLoadContext {
private final ArrayTupleBuilder btreeTupleBuilder;
- private final FrameTupleAppender btreeTupleAppender;
- private final FrameTupleAccessor btreeFrameTupleAccessor;
- private final FrameTupleReference btreeFrameTupleReference = new FrameTupleReference();
+ private final ArrayTupleReference btreeTupleReference;
private final float btreeFillFactor;
private IIndexBulkLoadContext btreeBulkLoadCtx;
private int currentInvListStartPageId;
private int currentInvListStartOffset;
- private final ByteArrayAccessibleOutputStream currentInvListTokenBaaos = new ByteArrayAccessibleOutputStream();
- private final FixedSizeTupleReference currentInvListToken = new FixedSizeTupleReference(invListTypeTraits);
+ private final ArrayTupleBuilder lastTupleBuilder;
+ private final ArrayTupleReference lastTuple;
private int currentPageId;
private ICachedPage currentPage;
- private final IInvertedListBuilder invListBuilder;
private final MultiComparator tokenCmp;
+ private final MultiComparator invListCmp;
- public BulkLoadContext(IInvertedListBuilder invListBuilder, int hyracksFrameSize, float btreeFillFactor) {
- this.invListBuilder = invListBuilder;
- this.tokenCmp = btree.getMultiComparator();
- this.btreeTupleBuffer = ByteBuffer.allocate(hyracksFrameSize);
+ public InvertedIndexBulkLoadContext(float btreeFillFactor) {
+ this.tokenCmp = MultiComparator.create(btree.getComparatorFactories());
+ this.invListCmp = MultiComparator.create(invListCmpFactories);
this.btreeTupleBuilder = new ArrayTupleBuilder(btree.getFieldCount());
- this.btreeTupleAppender = new FrameTupleAppender(hyracksFrameSize);
- // TODO: serde never used, only need correct number of fields
- // tuple contains (token, start page, end page, start offset, num
- // elements)
- RecordDescriptor recDesc = new RecordDescriptor(new ISerializerDeserializer[] {
- IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE });
- this.btreeFrameTupleAccessor = new FrameTupleAccessor(hyracksFrameSize, recDesc);
+ this.btreeTupleReference = new ArrayTupleReference();
this.btreeFillFactor = btreeFillFactor;
+ this.lastTupleBuilder = new ArrayTupleBuilder(numTokenFields + numInvListKeys);
+ this.lastTuple = new ArrayTupleReference();
}
- public void init(int startPageId, int fileId) throws HyracksDataException, TreeIndexException, PageAllocationException {
- btreeBulkLoadCtx = btree.beginBulkLoad(BTree.DEFAULT_FILL_FACTOR);
+ public void init(int startPageId, int fileId) throws HyracksDataException, TreeIndexException {
+ btreeBulkLoadCtx = btree.beginBulkLoad(btreeFillFactor);
currentPageId = startPageId;
currentPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), true);
currentPage.acquireWriteLatch();
invListBuilder.setTargetBuffer(currentPage.getBuffer().array(), 0);
- btreeFrameTupleAccessor.reset(btreeTupleBuffer);
}
public void deinit() throws HyracksDataException {
@@ -298,4 +256,101 @@
currentPage.acquireWriteLatch();
}
}
+
+ @Override
+ public IBufferCache getBufferCache() {
+ return bufferCache;
+ }
+
+ public int getInvListsFileId() {
+ return fileId;
+ }
+
+ public IBinaryComparatorFactory[] getInvListElementCmpFactories() {
+ return invListCmpFactories;
+ }
+
+ public ITypeTraits[] getTypeTraits() {
+ return invListTypeTraits;
+ }
+
+ public BTree getBTree() {
+ return btree;
+ }
+
+ public class InvertedIndexAccessor implements IIndexAccessor {
+ private final IInvertedIndexSearcher searcher;
+
+ public InvertedIndexAccessor(InvertedIndex index) {
+ this.searcher = new TOccurrenceSearcher(ctx, index);
+ }
+
+ @Override
+ public void insert(ITupleReference tuple) throws HyracksDataException, IndexException {
+ // TODO Auto-generated method stub
+ }
+
+ @Override
+ public void update(ITupleReference tuple) throws HyracksDataException, IndexException {
+ // TODO Auto-generated method stub
+ }
+
+ @Override
+ public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ // TODO Auto-generated method stub
+ }
+
+ @Override
+ public void upsert(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
+ // TODO Auto-generated method stub
+ }
+
+ @Override
+ public IIndexCursor createSearchCursor() {
+ return new InvertedIndexSearchCursor(searcher);
+ }
+
+ @Override
+ public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
+ IndexException {
+ searcher.search((InvertedIndexSearchCursor) cursor, (InvertedIndexSearchPredicate) searchPred);
+ }
+
+ public IInvertedIndexSearcher getSearcher() {
+ return searcher;
+ }
+ }
+
+ @Override
+ public IIndexAccessor createAccessor() {
+ return new InvertedIndexAccessor(this);
+ }
+
+ @Override
+ public IndexType getIndexType() {
+ return IndexType.INVERTED;
+ }
+
+ // This is just a dummy hyracks context for allocating frames for temporary
+ // results during inverted index searches.
+ // TODO: In the future we should use the real HyracksTaskContext to track
+ // frame usage.
+ private class DefaultHyracksCommonContext implements IHyracksCommonContext {
+ private final int FRAME_SIZE = 32768;
+
+ @Override
+ public int getFrameSize() {
+ return FRAME_SIZE;
+ }
+
+ @Override
+ public IIOManager getIOManager() {
+ return null;
+ }
+
+ @Override
+ public ByteBuffer allocateFrame() {
+ return ByteBuffer.allocate(FRAME_SIZE);
+ }
+ }
}
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexException.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexException.java
index 494ac0c..9762a74 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexException.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexException.java
@@ -15,7 +15,9 @@
package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
-public class InvertedIndexException extends Exception {
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+
+public class InvertedIndexException extends IndexException {
private static final long serialVersionUID = 1L;
public InvertedIndexException(String msg) {
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexSearchCursor.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexSearchCursor.java
new file mode 100644
index 0000000..1eeb576
--- /dev/null
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexSearchCursor.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearcher;
+
+public class InvertedIndexSearchCursor implements IIndexCursor {
+
+ private List<ByteBuffer> resultBuffers;
+ private int numResultBuffers;
+ private int currentBufferIndex = 0;
+ private int tupleIndex = 0;
+ private final IInvertedIndexSearcher invIndexSearcher;
+ private final IFrameTupleAccessor fta;
+ private final FixedSizeTupleReference resultTuple;
+
+ public InvertedIndexSearchCursor(IInvertedIndexSearcher invIndexSearcher) {
+ this.invIndexSearcher = invIndexSearcher;
+ this.fta = invIndexSearcher.createResultFrameTupleAccessor();
+ this.resultTuple = (FixedSizeTupleReference) invIndexSearcher.createResultTupleReference();
+ }
+
+ @Override
+ public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ currentBufferIndex = 0;
+ tupleIndex = 0;
+ resultBuffers = invIndexSearcher.getResultBuffers();
+ numResultBuffers = invIndexSearcher.getNumValidResultBuffers();
+ if (numResultBuffers > 0) {
+ fta.reset(resultBuffers.get(0));
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (currentBufferIndex < numResultBuffers && tupleIndex < fta.getTupleCount()) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void next() {
+ resultTuple.reset(fta.getBuffer().array(), fta.getTupleStartOffset(tupleIndex));
+ tupleIndex++;
+ if (tupleIndex >= fta.getTupleCount()) {
+ if (currentBufferIndex + 1 < numResultBuffers) {
+ currentBufferIndex++;
+ fta.reset(resultBuffers.get(currentBufferIndex));
+ tupleIndex = 0;
+ }
+ }
+ }
+
+ @Override
+ public ITupleReference getTuple() {
+ return resultTuple;
+ }
+
+ @Override
+ public void reset() {
+ currentBufferIndex = 0;
+ tupleIndex = 0;
+ invIndexSearcher.reset();
+ resultBuffers = invIndexSearcher.getResultBuffers();
+ numResultBuffers = invIndexSearcher.getNumValidResultBuffers();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ currentBufferIndex = 0;
+ tupleIndex = 0;
+ resultBuffers = null;
+ numResultBuffers = 0;
+ }
+}
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexSearchPredicate.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexSearchPredicate.java
new file mode 100644
index 0000000..9e9a2c3
--- /dev/null
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexSearchPredicate.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
+
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
+import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizer;
+
+public class InvertedIndexSearchPredicate implements ISearchPredicate {
+ private static final long serialVersionUID = 1L;
+
+ private ITupleReference queryTuple;
+ private int queryFieldIndex;
+ private final IBinaryTokenizer queryTokenizer;
+ private final IInvertedIndexSearchModifier searchModifier;
+
+ public InvertedIndexSearchPredicate(IBinaryTokenizer queryTokenizer, IInvertedIndexSearchModifier searchModifier) {
+ this.queryTokenizer = queryTokenizer;
+ this.searchModifier = searchModifier;
+ }
+
+ public void setQueryTuple(ITupleReference queryTuple) {
+ this.queryTuple = queryTuple;
+ }
+
+ public ITupleReference getQueryTuple() {
+ return queryTuple;
+ }
+
+ public void setQueryFieldIndex(int queryFieldIndex) {
+ this.queryFieldIndex = queryFieldIndex;
+ }
+
+ public int getQueryFieldIndex() {
+ return queryFieldIndex;
+ }
+
+ public IInvertedIndexSearchModifier getSearchModifier() {
+ return searchModifier;
+ }
+
+ public IBinaryTokenizer getQueryTokenizer() {
+ return queryTokenizer;
+ }
+
+ @Override
+ public MultiComparator getLowKeyComparator() {
+ // TODO: This doesn't make sense for an inverted index. Change ISearchPredicate interface.
+ return null;
+ }
+
+ @Override
+ public MultiComparator getHighKeyComparator() {
+ // TODO: This doesn't make sense for an inverted index. Change ISearchPredicate interface.
+ return null;
+ }
+}
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/SearchResultCursor.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/SearchResultCursor.java
deleted file mode 100644
index 5a7230c..0000000
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/SearchResultCursor.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
-
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexResultCursor;
-import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearcher;
-
-public class SearchResultCursor implements IInvertedIndexResultCursor {
-
- private List<ByteBuffer> resultBuffers;
- private IFrameTupleAccessor fta;
- private FixedSizeTupleReference resultTuple;
- private int numResultBuffers;
- private int currentBufferIndex = 0;
- private int tupleIndex = 0;
-
- public SearchResultCursor(IFrameTupleAccessor fta, ITupleReference resultTuple) {
- this.fta = fta;
- this.resultTuple = (FixedSizeTupleReference) resultTuple;
- }
-
- @Override
- public boolean hasNext() {
- if (currentBufferIndex < numResultBuffers && tupleIndex < fta.getTupleCount())
- return true;
- else
- return false;
- }
-
- @Override
- public void next() {
- resultTuple.reset(fta.getBuffer().array(), fta.getTupleStartOffset(tupleIndex));
- tupleIndex++;
- if (tupleIndex >= fta.getTupleCount()) {
- if (currentBufferIndex + 1 < numResultBuffers) {
- currentBufferIndex++;
- fta.reset(resultBuffers.get(currentBufferIndex));
- tupleIndex = 0;
- }
- }
- }
-
- @Override
- public ITupleReference getTuple() {
- return resultTuple;
- }
-
- @Override
- public void reset(IInvertedIndexSearcher invIndexSearcher) {
- currentBufferIndex = 0;
- tupleIndex = 0;
- resultBuffers = invIndexSearcher.getResultBuffers();
- numResultBuffers = invIndexSearcher.getNumValidResultBuffers();
- if (numResultBuffers > 0) {
- fta.reset(resultBuffers.get(0));
- }
- }
-}
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
index b24d416..1f886a2 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
@@ -22,7 +22,7 @@
import java.util.List;
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.context.IHyracksCommonContext;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
@@ -41,17 +41,19 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexResultCursor;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearcher;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListCursor;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizer;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IToken;
+// TODO: The search procedure is rather confusing regarding cursor positions, hasNext() calls etc.
+// Needs an overhaul some time.
public class TOccurrenceSearcher implements IInvertedIndexSearcher {
- protected final IHyracksTaskContext ctx;
+ protected final IHyracksCommonContext ctx;
protected final FixedSizeFrameTupleAppender resultFrameTupleApp;
protected final FixedSizeFrameTupleAccessor resultFrameTupleAcc;
protected final FixedSizeTupleReference resultTuple;
@@ -67,7 +69,7 @@
protected final ITreeIndexFrame interiorFrame;
protected final ITreeIndexCursor btreeCursor;
protected final FrameTupleReference searchKey = new FrameTupleReference();
- protected final RangePredicate btreePred = new RangePredicate(true, null, null, true, true, null, null);
+ protected final RangePredicate btreePred = new RangePredicate(null, null, true, true, null, null);
protected final ITreeIndexAccessor btreeAccessor;
protected RecordDescriptor queryTokenRecDesc = new RecordDescriptor(
@@ -78,7 +80,7 @@
protected ByteBuffer queryTokenFrame;
protected final InvertedIndex invIndex;
- protected final IBinaryTokenizer queryTokenizer;
+ protected final MultiComparator invListCmp;
protected final ITypeTraits[] invListFieldsWithCount;
protected int occurrenceThreshold;
@@ -86,10 +88,10 @@
protected List<IInvertedListCursor> invListCursorCache = new ArrayList<IInvertedListCursor>(cursorCacheSize);
protected List<IInvertedListCursor> invListCursors = new ArrayList<IInvertedListCursor>(cursorCacheSize);
- public TOccurrenceSearcher(IHyracksTaskContext ctx, InvertedIndex invIndex, IBinaryTokenizer queryTokenizer) {
+ public TOccurrenceSearcher(IHyracksCommonContext ctx, InvertedIndex invIndex) {
this.ctx = ctx;
this.invIndex = invIndex;
- this.queryTokenizer = queryTokenizer;
+ this.invListCmp = MultiComparator.create(invIndex.getInvListElementCmpFactories());
leafFrame = invIndex.getBTree().getLeafFrameFactory().createFrame();
interiorFrame = invIndex.getBTree().getInteriorFrameFactory().createFrame();
@@ -112,7 +114,7 @@
newResultBuffers.add(ctx.allocateFrame());
prevResultBuffers.add(ctx.allocateFrame());
- MultiComparator searchCmp = invIndex.getBTree().getMultiComparator();
+ MultiComparator searchCmp = MultiComparator.create(invIndex.getBTree().getComparatorFactories());
btreePred.setLowKeyComparator(searchCmp);
btreePred.setHighKeyComparator(searchCmp);
btreePred.setLowKey(searchKey, true);
@@ -141,12 +143,15 @@
currentNumResults = 0;
}
- public void search(IInvertedIndexResultCursor resultCursor, ITupleReference queryTuple, int queryField,
- IInvertedIndexSearchModifier searchModifier) throws Exception {
-
- queryTokenAppender.reset(queryTokenFrame, true);
- queryTokenizer.reset(queryTuple.getFieldData(queryField), queryTuple.getFieldStart(queryField),
- queryTuple.getFieldLength(queryField));
+ public void search(InvertedIndexSearchCursor resultCursor, InvertedIndexSearchPredicate searchPred) throws HyracksDataException, IndexException {
+ ITupleReference queryTuple = searchPred.getQueryTuple();
+ int queryFieldIndex = searchPred.getQueryFieldIndex();
+ IInvertedIndexSearchModifier searchModifier = searchPred.getSearchModifier();
+ IBinaryTokenizer queryTokenizer = searchPred.getQueryTokenizer();
+
+ queryTokenAppender.reset(queryTokenFrame, true);
+ queryTokenizer.reset(queryTuple.getFieldData(queryFieldIndex), queryTuple.getFieldStart(queryFieldIndex),
+ queryTuple.getFieldLength(queryFieldIndex));
while (queryTokenizer.hasNext()) {
queryTokenizer.next();
@@ -194,10 +199,10 @@
maxResultBufIdx = mergePrefixLists(numPrefixLists, numQueryTokens);
maxResultBufIdx = mergeSuffixLists(numPrefixLists, numQueryTokens, maxResultBufIdx);
- resultCursor.reset(this);
+ resultCursor.open(null, searchPred);
}
- protected int mergePrefixLists(int numPrefixTokens, int numQueryTokens) throws IOException {
+ protected int mergePrefixLists(int numPrefixTokens, int numQueryTokens) throws HyracksDataException {
int maxPrevBufIdx = 0;
for (int i = 0; i < numPrefixTokens; i++) {
swap = prevResultBuffers;
@@ -212,7 +217,7 @@
return maxPrevBufIdx;
}
- protected int mergeSuffixLists(int numPrefixTokens, int numQueryTokens, int maxPrevBufIdx) throws IOException {
+ protected int mergeSuffixLists(int numPrefixTokens, int numQueryTokens, int maxPrevBufIdx) throws HyracksDataException {
for (int i = numPrefixTokens; i < numQueryTokens; i++) {
swap = prevResultBuffers;
prevResultBuffers = newResultBuffers;
@@ -234,7 +239,7 @@
}
protected int mergeSuffixListProbe(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers,
- int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) throws IOException {
+ int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) {
int newBufIdx = 0;
ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
@@ -246,8 +251,6 @@
currentNumResults = 0;
- MultiComparator invListCmp = invIndex.getInvListElementCmp();
-
resultFrameTupleAcc.reset(prevCurrentBuffer);
resultFrameTupleApp.reset(newCurrentBuffer, true);
@@ -281,7 +284,7 @@
}
protected int mergeSuffixListScan(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers,
- int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) throws IOException {
+ int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) {
int newBufIdx = 0;
ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
@@ -292,8 +295,6 @@
boolean advancePrevResult = false;
int resultTidx = 0;
- MultiComparator invListCmp = invIndex.getInvListElementCmp();
-
resultFrameTupleAcc.reset(prevCurrentBuffer);
resultFrameTupleApp.reset(newCurrentBuffer, true);
@@ -345,7 +346,9 @@
if (advanceCursor) {
invListTidx++;
- invListCursor.next();
+ if (invListCursor.hasNext()) {
+ invListCursor.next();
+ }
}
}
@@ -375,7 +378,7 @@
}
protected int mergePrefixList(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers,
- int maxPrevBufIdx, List<ByteBuffer> newResultBuffers) throws IOException {
+ int maxPrevBufIdx, List<ByteBuffer> newResultBuffers) {
int newBufIdx = 0;
ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
@@ -386,8 +389,6 @@
boolean advancePrevResult = false;
int resultTidx = 0;
- MultiComparator invListCmp = invIndex.getInvListElementCmp();
-
resultFrameTupleAcc.reset(prevCurrentBuffer);
resultFrameTupleApp.reset(newCurrentBuffer, true);
@@ -438,7 +439,9 @@
if (advanceCursor) {
invListTidx++;
- invListCursor.next();
+ if (invListCursor.hasNext()) {
+ invListCursor.next();
+ }
}
}
@@ -447,7 +450,9 @@
ITupleReference invListTuple = invListCursor.getTuple();
newBufIdx = appendTupleToNewResults(invListTuple, 1, newBufIdx);
invListTidx++;
- invListCursor.next();
+ if (invListCursor.hasNext()) {
+ invListCursor.next();
+ }
}
// append remaining elements from previous result set
@@ -473,7 +478,7 @@
return newBufIdx;
}
- protected int appendTupleToNewResults(ITupleReference tuple, int newCount, int newBufIdx) throws IOException {
+ protected int appendTupleToNewResults(ITupleReference tuple, int newCount, int newBufIdx) {
ByteBuffer newCurrentBuffer = newResultBuffers.get(newBufIdx);
if (!resultFrameTupleApp.hasSpace()) {
@@ -523,7 +528,7 @@
public int getOccurrenceThreshold() {
return occurrenceThreshold;
}
-
+
public void printNewResults(int maxResultBufIdx) {
StringBuffer strBuffer = new StringBuffer();
for (int i = 0; i <= maxResultBufIdx; i++) {
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixProbeOnly.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixProbeOnly.java
index 0f5439b..957ecc0 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixProbeOnly.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixProbeOnly.java
@@ -15,24 +15,25 @@
package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
-import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListCursor;
-import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizer;
public class TOccurrenceSearcherSuffixProbeOnly extends TOccurrenceSearcher {
- public TOccurrenceSearcherSuffixProbeOnly(IHyracksTaskContext ctx, InvertedIndex invIndex,
- IBinaryTokenizer queryTokenizer) {
- super(ctx, invIndex, queryTokenizer);
+ protected final MultiComparator invListCmp;
+
+ public TOccurrenceSearcherSuffixProbeOnly(IHyracksTaskContext ctx, InvertedIndex invIndex) {
+ super(ctx, invIndex);
+ this.invListCmp = MultiComparator.create(invIndex.getInvListElementCmpFactories());
}
- protected int mergeSuffixLists(int numPrefixTokens, int numQueryTokens, int maxPrevBufIdx) throws IOException {
+ protected int mergeSuffixLists(int numPrefixTokens, int numQueryTokens, int maxPrevBufIdx) throws HyracksDataException {
for (int i = numPrefixTokens; i < numQueryTokens; i++) {
swap = prevResultBuffers;
prevResultBuffers = newResultBuffers;
@@ -48,7 +49,7 @@
}
protected int mergeSuffixListProbe(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers,
- int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) throws IOException {
+ int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) {
int newBufIdx = 0;
ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
@@ -58,8 +59,6 @@
int resultTidx = 0;
- MultiComparator invListCmp = invIndex.getInvListElementCmp();
-
resultFrameTupleAcc.reset(prevCurrentBuffer);
resultFrameTupleApp.reset(newCurrentBuffer, true);
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixScanOnly.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixScanOnly.java
index b997dc9..bd9bd60 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixScanOnly.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixScanOnly.java
@@ -15,25 +15,26 @@
package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
-import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListCursor;
-import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizer;
public class TOccurrenceSearcherSuffixScanOnly extends TOccurrenceSearcher {
- public TOccurrenceSearcherSuffixScanOnly(IHyracksTaskContext ctx, InvertedIndex invIndex,
- IBinaryTokenizer queryTokenizer) {
- super(ctx, invIndex, queryTokenizer);
+ protected final MultiComparator invListCmp;
+
+ public TOccurrenceSearcherSuffixScanOnly(IHyracksTaskContext ctx, InvertedIndex invIndex) {
+ super(ctx, invIndex);
+ this.invListCmp = MultiComparator.create(invIndex.getInvListElementCmpFactories());
}
- protected int mergeSuffixLists(int numPrefixTokens, int numQueryTokens, int maxPrevBufIdx) throws IOException {
+ protected int mergeSuffixLists(int numPrefixTokens, int numQueryTokens, int maxPrevBufIdx) throws HyracksDataException {
for (int i = numPrefixTokens; i < numQueryTokens; i++) {
swap = prevResultBuffers;
prevResultBuffers = newResultBuffers;
@@ -49,7 +50,7 @@
}
protected int mergeSuffixListScan(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers,
- int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) throws IOException {
+ int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) {
int newBufIdx = 0;
ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
@@ -61,8 +62,6 @@
boolean advancePrevResult = false;
int resultTidx = 0;
- MultiComparator invListCmp = invIndex.getInvListElementCmp();
-
resultFrameTupleAcc.reset(prevCurrentBuffer);
resultFrameTupleApp.reset(newCurrentBuffer, true);
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/JaccardSearchModifier.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/JaccardSearchModifier.java
index f42841c..0a4961d 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/JaccardSearchModifier.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/JaccardSearchModifier.java
@@ -31,7 +31,7 @@
@Override
public int getOccurrenceThreshold(List<IInvertedListCursor> invListCursors) {
- return (int) Math.floor((float) invListCursors.size() * jaccThresh);
+ return Math.max((int) Math.floor((float) invListCursors.size() * jaccThresh), 1);
}
@Override
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/AbstractUTF8StringBinaryTokenizer.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/AbstractUTF8StringBinaryTokenizer.java
index be6bfb0..c2992f5 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/AbstractUTF8StringBinaryTokenizer.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/AbstractUTF8StringBinaryTokenizer.java
@@ -63,7 +63,7 @@
if (sourceHasTypeTag) {
index++; // skip type tag
}
- utf8Length = UTF8StringPointable.getUTFLen(data, index);
+ utf8Length = UTF8StringPointable.getUTFLength(data, index);
index += 2; // skip utf8 length indicator
this.data = data;
this.length = length + start;
diff --git a/hyracks/hyracks-storage-am-rtree/pom.xml b/hyracks/hyracks-storage-am-rtree/pom.xml
index cd72ed7..abdb9d5 100644
--- a/hyracks/hyracks-storage-am-rtree/pom.xml
+++ b/hyracks/hyracks-storage-am-rtree/pom.xml
@@ -46,13 +46,6 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>edu.uci.ics.hyracks</groupId>
- <artifactId>hyracks-control-nc</artifactId>
- <version>0.2.1-SNAPSHOT</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.8.1</version>
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeFrame.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeFrame.java
index dd57986..cad7981 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeFrame.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeFrame.java
@@ -15,25 +15,21 @@
package edu.uci.ics.hyracks.storage.am.rtree.api;
-import edu.uci.ics.hyracks.storage.am.common.api.ISplitKey;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
public interface IRTreeFrame extends ITreeIndexFrame {
- public void computeMBR(ISplitKey splitKey);
+ public void delete(int tupleIndex, MultiComparator cmp);
- public void delete(int tupleIndex, MultiComparator cmp);
+ public long getPageNsn();
- public long getPageNsn();
+ public void setPageNsn(long pageNsn);
- public void setPageNsn(long pageNsn);
+ public int getRightPage();
- public int getRightPage();
+ public void setRightPage(int rightPage);
- public void setRightPage(int rightPage);
-
- public void adjustMBR(ITreeIndexTupleReference[] tuples);
+ public void adjustMBR();
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java
index 2b3065d..5f333f3 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java
@@ -16,28 +16,29 @@
package edu.uci.ics.hyracks.storage.am.rtree.api;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.rtree.impls.PathList;
public interface IRTreeInteriorFrame extends IRTreeFrame {
- public boolean findBestChild(ITupleReference tuple, MultiComparator cmp);
+ public boolean findBestChild(ITupleReference tuple, MultiComparator cmp);
- public int getBestChildPageId();
+ public int getBestChildPageId();
- public int getChildPageIdIfIntersect(ITupleReference tuple, int tupleIndex,
- MultiComparator cmp);
+ public int getChildPageId(int tupleIndex);
- public int findTupleByPointer(ITupleReference tuple, MultiComparator cmp);
+ public int getChildPageIdIfIntersect(ITupleReference tuple, int tupleIndex, MultiComparator cmp);
- public int findTupleByPointer(ITupleReference tuple, PathList traverseList,
- int parentId, MultiComparator cmp);
+ public int findTupleByPointer(ITupleReference tuple, MultiComparator cmp);
- public void adjustKey(ITupleReference tuple, int tupleIndex,
- MultiComparator cmp);
+ public int findTupleByPointer(ITupleReference tuple, PathList traverseList, int parentIndex, MultiComparator cmp);
- public boolean recomputeMBR(ITupleReference tuple, int tupleIndex,
- MultiComparator cmp);
+ public void adjustKey(ITupleReference tuple, int tupleIndex, MultiComparator cmp) throws TreeIndexException;
- public void enlarge(ITupleReference tuple, MultiComparator cmp);
+ public boolean recomputeMBR(ITupleReference tuple, int tupleIndex, MultiComparator cmp);
+
+ public void enlarge(ITupleReference tuple, MultiComparator cmp);
+
+ boolean checkEnlargement(ITupleReference tuple, MultiComparator cmp);
}
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java
index 51abb79..0470da9 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java
@@ -17,33 +17,26 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.common.util.IndexUtils;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
public class RTreeDataflowHelper extends TreeIndexDataflowHelper {
+ private final IPrimitiveValueProviderFactory[] valueProviderFactories;
+
public RTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- boolean createIfNotExists) {
- super(opDesc, ctx, partition, createIfNotExists);
+ IPrimitiveValueProviderFactory[] valueProviderFactories) {
+ super(opDesc, ctx, partition);
+ this.valueProviderFactories = valueProviderFactories;
}
@Override
public ITreeIndex createIndexInstance() throws HyracksDataException {
- MultiComparator cmp = IndexUtils.createMultiComparator(treeOpDesc.getTreeIndexComparatorFactories());
- IBufferCache bufferCache = treeOpDesc.getStorageManager().getBufferCache(ctx);
- ITreeIndexMetaDataFrameFactory metaDataFrameFactory = new LIFOMetaDataFrameFactory();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, indexFileId, 0,
- metaDataFrameFactory);
- return new RTree(bufferCache, treeOpDesc.getTreeIndexTypeTraits().length, cmp, freePageManager,
- treeOpDesc.getTreeIndexInteriorFactory(), treeOpDesc.getTreeIndexLeafFactory());
+ return RTreeUtils.createRTree(treeOpDesc.getStorageManager().getBufferCache(ctx),
+ treeOpDesc.getTreeIndexTypeTraits(), valueProviderFactories,
+ treeOpDesc.getTreeIndexComparatorFactories());
}
}
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelperFactory.java
index 8cd4e95..6b9fd4c 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelperFactory.java
@@ -16,6 +16,7 @@
package edu.uci.ics.hyracks.storage.am.rtree.dataflow;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
@@ -24,9 +25,15 @@
private static final long serialVersionUID = 1L;
+ private final IPrimitiveValueProviderFactory[] valueProviderFactories;
+
+ public RTreeDataflowHelperFactory(IPrimitiveValueProviderFactory[] valueProviderFactories) {
+ this.valueProviderFactories = valueProviderFactories;
+ }
+
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition, boolean createIfNotExists) {
- return new RTreeDataflowHelper(opDesc, ctx, partition, createIfNotExists);
+ int partition) {
+ return new RTreeDataflowHelper(opDesc, ctx, partition, valueProviderFactories);
}
}
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorDescriptor.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorDescriptor.java
index 8cb4e55..9818dce 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorDescriptor.java
@@ -21,9 +21,9 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
@@ -34,22 +34,22 @@
private static final long serialVersionUID = 1L;
- private int[] keyFields; // fields in input tuple to be used as keys
+ protected int[] keyFields; // fields in input tuple to be used as keys
- public RTreeSearchOperatorDescriptor(JobSpecification spec, RecordDescriptor recDesc,
+ public RTreeSearchOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc,
IStorageManagerInterface storageManager, IIndexRegistryProvider<IIndex> indexRegistryProvider,
- IFileSplitProvider fileSplitProvider, ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, ITypeTraits[] typeTraits,
+ IFileSplitProvider fileSplitProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] comparatorFactories, int[] keyFields,
- IIndexDataflowHelperFactory dataflowHelperFactory) {
- super(spec, 1, 1, recDesc, storageManager, indexRegistryProvider, fileSplitProvider, interiorFrameFactory,
- leafFrameFactory, typeTraits, comparatorFactories, dataflowHelperFactory);
+ IIndexDataflowHelperFactory dataflowHelperFactory, boolean retainInput, IOperationCallbackProvider opCallbackProvider) {
+ super(spec, 1, 1, recDesc, storageManager, indexRegistryProvider, fileSplitProvider, typeTraits,
+ comparatorFactories, dataflowHelperFactory, null, false, opCallbackProvider);
this.keyFields = keyFields;
}
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
- return new RTreeSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, keyFields);
+ return new RTreeSearchOperatorNodePushable(this, ctx, opCallbackProvider, partition, recordDescProvider,
+ keyFields);
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
index 9f62b8f..3781037 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
@@ -15,58 +15,25 @@
package edu.uci.ics.hyracks.storage.am.rtree.dataflow;
-import java.io.DataOutput;
-import java.nio.ByteBuffer;
-
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallbackProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.PermutingFrameTupleReference;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexSearchOperatorNodePushable;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
-import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeSearchCursor;
import edu.uci.ics.hyracks.storage.am.rtree.impls.SearchPredicate;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
-public class RTreeSearchOperatorNodePushable extends AbstractUnaryInputUnaryOutputOperatorNodePushable {
- private TreeIndexDataflowHelper treeIndexHelper;
- private FrameTupleAccessor accessor;
-
- private ByteBuffer writeBuffer;
- private FrameTupleAppender appender;
- private ArrayTupleBuilder tb;
- private DataOutput dos;
-
- private RTree rtree;
- private PermutingFrameTupleReference searchKey;
- private SearchPredicate searchPred;
- private MultiComparator cmp;
- private ITreeIndexCursor cursor;
- private ITreeIndexFrame interiorFrame;
- private ITreeIndexFrame leafFrame;
- private ITreeIndexAccessor indexAccessor;
-
- private RecordDescriptor recDesc;
+public class RTreeSearchOperatorNodePushable extends TreeIndexSearchOperatorNodePushable {
+ protected PermutingFrameTupleReference searchKey;
+ protected MultiComparator cmp;
public RTreeSearchOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition, IRecordDescriptorProvider recordDescProvider, int[] keyFields) {
- treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(
- opDesc, ctx, partition, false);
- this.recDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getOperatorId(), 0);
+ IOperationCallbackProvider opCallbackProvider, int partition, IRecordDescriptorProvider recordDescProvider,
+ int[] keyFields) {
+ super(opDesc, ctx, partition, recordDescProvider);
if (keyFields != null && keyFields.length > 0) {
searchKey = new PermutingFrameTupleReference();
searchKey.setFieldPermutation(keyFields);
@@ -74,101 +41,15 @@
}
@Override
- public void open() throws HyracksDataException {
- AbstractTreeIndexOperatorDescriptor opDesc = (AbstractTreeIndexOperatorDescriptor) treeIndexHelper
- .getOperatorDescriptor();
- accessor = new FrameTupleAccessor(treeIndexHelper.getHyracksTaskContext().getFrameSize(), recDesc);
- interiorFrame = opDesc.getTreeIndexInteriorFactory().createFrame();
- leafFrame = opDesc.getTreeIndexLeafFactory().createFrame();
- cursor = new RTreeSearchCursor((IRTreeInteriorFrame) interiorFrame, (IRTreeLeafFrame) leafFrame);
- try {
- treeIndexHelper.init();
- writer.open();
- try {
- rtree = (RTree) treeIndexHelper.getIndex();
- int keySearchFields = rtree.getCmp().getComparators().length;
- IBinaryComparator[] keySearchComparators = new IBinaryComparator[keySearchFields];
- for (int i = 0; i < keySearchFields; i++) {
- keySearchComparators[i] = rtree.getCmp().getComparators()[i];
- }
- cmp = new MultiComparator(keySearchComparators);
- searchPred = new SearchPredicate(searchKey, cmp);
- writeBuffer = treeIndexHelper.getHyracksTaskContext().allocateFrame();
- tb = new ArrayTupleBuilder(rtree.getFieldCount());
- dos = tb.getDataOutput();
- appender = new FrameTupleAppender(treeIndexHelper.getHyracksTaskContext().getFrameSize());
- appender.reset(writeBuffer, true);
- indexAccessor = rtree.createAccessor();
- } catch (Exception e) {
- writer.fail();
- throw e;
- }
-
- } catch (Exception e) {
- treeIndexHelper.deinit();
- throw new HyracksDataException(e);
- }
- }
-
- private void writeSearchResults() throws Exception {
- while (cursor.hasNext()) {
- tb.reset();
- cursor.next();
-
- ITupleReference frameTuple = cursor.getTuple();
- for (int i = 0; i < frameTuple.getFieldCount(); i++) {
- dos.write(frameTuple.getFieldData(i), frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
- tb.addFieldEndOffset();
- }
-
- if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
- FrameUtils.flushFrame(writeBuffer, writer);
- appender.reset(writeBuffer, true);
- if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
- throw new IllegalStateException();
- }
- }
- }
+ protected ISearchPredicate createSearchPredicate() {
+ cmp = RTreeUtils.getSearchMultiComparator(treeIndex.getComparatorFactories(), searchKey);
+ return new SearchPredicate(searchKey, cmp);
}
@Override
- public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
- accessor.reset(buffer);
-
- int tupleCount = accessor.getTupleCount();
- try {
- for (int i = 0; i < tupleCount; i++) {
- searchKey.reset(accessor, i);
-
- searchPred.setSearchKey(searchKey);
- cursor.reset();
- indexAccessor.search(cursor, searchPred);
- writeSearchResults();
- }
- } catch (Exception e) {
- throw new HyracksDataException(e);
+ protected void resetSearchPredicate(int tupleIndex) {
+ if (searchKey != null) {
+ searchKey.reset(accessor, tupleIndex);
}
}
-
- @Override
- public void close() throws HyracksDataException {
- try {
- if (appender.getTupleCount() > 0) {
- FrameUtils.flushFrame(writeBuffer, writer);
- }
- writer.close();
- try {
- cursor.close();
- } catch (Exception e) {
- throw new HyracksDataException(e);
- }
- } finally {
- treeIndexHelper.deinit();
- }
- }
-
- @Override
- public void fail() throws HyracksDataException {
- writer.fail();
- }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
index 546a4d4..84e66ef 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
@@ -15,200 +15,319 @@
package edu.uci.ics.hyracks.storage.am.rtree.frames;
-import java.util.ArrayList;
-
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProvider;
import edu.uci.ics.hyracks.storage.am.common.api.ISplitKey;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.frames.TreeIndexNSMFrame;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.EntriesOrder;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeSplitKey;
import edu.uci.ics.hyracks.storage.am.rtree.impls.Rectangle;
import edu.uci.ics.hyracks.storage.am.rtree.impls.TupleEntryArrayList;
import edu.uci.ics.hyracks.storage.am.rtree.impls.UnorderedSlotManager;
import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriter;
-public abstract class RTreeNSMFrame extends TreeIndexNSMFrame implements
- IRTreeFrame {
- protected static final int pageNsnOff = smFlagOff + 1;
- protected static final int rightPageOff = pageNsnOff + 8;
+public abstract class RTreeNSMFrame extends TreeIndexNSMFrame implements IRTreeFrame {
+ protected static final int pageNsnOff = smFlagOff + 1;
+ protected static final int rightPageOff = pageNsnOff + 8;
- protected ITreeIndexTupleReference[] tuples;
- protected ITreeIndexTupleReference cmpFrameTuple;
- protected TupleEntryArrayList tupleEntries1; // used for split and checking
- // enlargement
- protected TupleEntryArrayList tupleEntries2; // used for split
+ protected ITreeIndexTupleReference[] tuples;
+ protected ITreeIndexTupleReference cmpFrameTuple;
+ protected TupleEntryArrayList tupleEntries1; // used for split and checking
+ // enlargement
+ protected TupleEntryArrayList tupleEntries2; // used for split
- protected Rectangle[] rec;
+ protected Rectangle[] rec;
- protected static final double splitFactor = 0.4;
- protected static final int nearMinimumOverlapFactor = 32;
- private static final double doubleEpsilon = computeDoubleEpsilon();
- private static final int numTuplesEntries = 100;
- protected final IPrimitiveValueProvider[] keyValueProviders;
-
- public RTreeNSMFrame(ITreeIndexTupleWriter tupleWriter, IPrimitiveValueProvider[] keyValueProviders) {
- super(tupleWriter, new UnorderedSlotManager());
- this.tuples = new ITreeIndexTupleReference[keyValueProviders.length];
- for (int i = 0; i < keyValueProviders.length; i++) {
- this.tuples[i] = tupleWriter.createTupleReference();
- }
- cmpFrameTuple = tupleWriter.createTupleReference();
+ protected static final double splitFactor = 0.4;
+ protected static final int nearMinimumOverlapFactor = 32;
+ private static final double doubleEpsilon = computeDoubleEpsilon();
+ private static final int numTuplesEntries = 100;
+ protected final IPrimitiveValueProvider[] keyValueProviders;
- tupleEntries1 = new TupleEntryArrayList(numTuplesEntries,
- numTuplesEntries);
- tupleEntries2 = new TupleEntryArrayList(numTuplesEntries,
- numTuplesEntries);
- rec = new Rectangle[4];
- for (int i = 0; i < 4; i++) {
- rec[i] = new Rectangle(keyValueProviders.length / 2);
- }
- this.keyValueProviders = keyValueProviders;
- }
+ public RTreeNSMFrame(ITreeIndexTupleWriter tupleWriter, IPrimitiveValueProvider[] keyValueProviders) {
+ super(tupleWriter, new UnorderedSlotManager());
+ this.tuples = new ITreeIndexTupleReference[keyValueProviders.length];
+ for (int i = 0; i < keyValueProviders.length; i++) {
+ this.tuples[i] = tupleWriter.createTupleReference();
+ }
+ cmpFrameTuple = tupleWriter.createTupleReference();
- private static double computeDoubleEpsilon() {
- double doubleEpsilon = 1.0;
+ tupleEntries1 = new TupleEntryArrayList(numTuplesEntries, numTuplesEntries);
+ tupleEntries2 = new TupleEntryArrayList(numTuplesEntries, numTuplesEntries);
+ rec = new Rectangle[4];
+ for (int i = 0; i < 4; i++) {
+ rec[i] = new Rectangle(keyValueProviders.length / 2);
+ }
+ this.keyValueProviders = keyValueProviders;
+ }
- do {
- doubleEpsilon /= 2.0;
- } while (1.0 + (doubleEpsilon / 2.0) != 1.0);
- return doubleEpsilon;
- }
+ private static double computeDoubleEpsilon() {
+ double doubleEpsilon = 1.0;
- public static double doubleEpsilon() {
- return doubleEpsilon;
- }
+ do {
+ doubleEpsilon /= 2.0;
+ } while (1.0 + (doubleEpsilon / 2.0) != 1.0);
+ return doubleEpsilon;
+ }
- @Override
- public void initBuffer(byte level) {
- super.initBuffer(level);
- buf.putLong(pageNsnOff, 0);
- buf.putInt(rightPageOff, -1);
- }
+ public static double doubleEpsilon() {
+ return doubleEpsilon;
+ }
- public void setTupleCount(int tupleCount) {
- buf.putInt(tupleCountOff, tupleCount);
- }
+ @Override
+ public void initBuffer(byte level) {
+ super.initBuffer(level);
+ buf.putLong(pageNsnOff, 0);
+ buf.putInt(rightPageOff, -1);
+ }
- @Override
- public void setPageNsn(long pageNsn) {
- buf.putLong(pageNsnOff, pageNsn);
- }
+ public void setTupleCount(int tupleCount) {
+ buf.putInt(tupleCountOff, tupleCount);
+ }
- @Override
- public long getPageNsn() {
- return buf.getLong(pageNsnOff);
- }
+ @Override
+ public void setPageNsn(long pageNsn) {
+ buf.putLong(pageNsnOff, pageNsn);
+ }
- @Override
- protected void resetSpaceParams() {
- buf.putInt(freeSpaceOff, rightPageOff + 4);
- buf.putInt(totalFreeSpaceOff, buf.capacity() - (rightPageOff + 4));
- }
+ @Override
+ public long getPageNsn() {
+ return buf.getLong(pageNsnOff);
+ }
- @Override
- public int getRightPage() {
- return buf.getInt(rightPageOff);
- }
+ @Override
+ protected void resetSpaceParams() {
+ buf.putInt(freeSpaceOff, rightPageOff + 4);
+ buf.putInt(totalFreeSpaceOff, buf.capacity() - (rightPageOff + 4));
+ }
- @Override
- public void setRightPage(int rightPage) {
- buf.putInt(rightPageOff, rightPage);
- }
+ @Override
+ public int getRightPage() {
+ return buf.getInt(rightPageOff);
+ }
- protected ITreeIndexTupleReference[] getTuples() {
- return tuples;
- }
+ @Override
+ public void setRightPage(int rightPage) {
+ buf.putInt(rightPageOff, rightPage);
+ }
- // for debugging
- public ArrayList<Integer> getChildren(MultiComparator cmp) {
- ArrayList<Integer> ret = new ArrayList<Integer>();
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
- int tupleCount = buf.getInt(tupleCountOff);
- for (int i = 0; i < tupleCount; i++) {
- int tupleOff = slotManager.getTupleOff(slotManager.getSlotOff(i));
- frameTuple.resetByTupleOffset(buf, tupleOff);
+ protected ITreeIndexTupleReference[] getTuples() {
+ return tuples;
+ }
- int intVal = IntegerSerializerDeserializer.getInt(
- buf.array(),
- frameTuple.getFieldStart(frameTuple.getFieldCount() - 1)
- + frameTuple.getFieldLength(frameTuple
- .getFieldCount() - 1));
- ret.add(intVal);
- }
- return ret;
- }
+ @Override
+ public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey) throws TreeIndexException {
+ RTreeSplitKey rTreeSplitKey = ((RTreeSplitKey) splitKey);
+ RTreeTypeAwareTupleWriter rTreeTupleWriterLeftFrame = ((RTreeTypeAwareTupleWriter) tupleWriter);
+ RTreeTypeAwareTupleWriter rTreeTupleWriterRightFrame = ((RTreeTypeAwareTupleWriter) rightFrame.getTupleWriter());
- public void generateDist(ITupleReference tuple,
- TupleEntryArrayList entries, Rectangle rec, int start, int end) {
- int j = 0;
- while (entries.get(j).getTupleIndex() == -1) {
- j++;
- }
- frameTuple.resetByTupleIndex(this, entries.get(j).getTupleIndex());
- rec.set(frameTuple, keyValueProviders);
- for (int i = start; i < end; ++i) {
- if (i != j) {
- if (entries.get(i).getTupleIndex() != -1) {
- frameTuple.resetByTupleIndex(this, entries.get(i)
- .getTupleIndex());
- rec.enlarge(frameTuple, keyValueProviders);
- } else {
- rec.enlarge(tuple, keyValueProviders);
- }
- }
- }
- }
+ // calculations are based on the R*-tree paper
+ int m = (int) Math.floor((getTupleCount() + 1) * splitFactor);
+ int splitDistribution = getTupleCount() - (2 * m) + 2;
- @Override
- public ITreeIndexTupleReference createTupleReference() {
- return tupleWriter.createTupleReference();
- }
+ // to calculate the minimum margin in order to pick the split axis
+ double minMargin = Double.MAX_VALUE;
+ int splitAxis = 0, sortOrder = 0;
- public void adjustMBRImpl(ITreeIndexTupleReference[] tuples) {
- int maxFieldPos = keyValueProviders.length / 2;
- for (int i = 1; i < getTupleCount(); i++) {
- frameTuple.resetByTupleIndex(this, i);
- for (int j = 0; j < maxFieldPos; j++) {
- int k = maxFieldPos + j;
- double valA = keyValueProviders[j].getValue(frameTuple.getFieldData(j), frameTuple.getFieldStart(j));
- double valB = keyValueProviders[j].getValue(tuples[j].getFieldData(j), tuples[j].getFieldStart(j));
- if (valA < valB) {
- tuples[j].resetByTupleIndex(this, i);
- }
- valA = keyValueProviders[k].getValue(frameTuple.getFieldData(k), frameTuple.getFieldStart(k));
- valB = keyValueProviders[k].getValue(tuples[k].getFieldData(k), tuples[k].getFieldStart(k));
- if (valA > valB) {
- tuples[k].resetByTupleIndex(this, i);
- }
- }
- }
- }
+ int maxFieldPos = keyValueProviders.length / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ for (int k = 0; k < getTupleCount(); ++k) {
- @Override
- public void computeMBR(ISplitKey splitKey) {
- RTreeSplitKey rTreeSplitKey = ((RTreeSplitKey) splitKey);
- RTreeTypeAwareTupleWriter rTreeTupleWriterLeftFrame = ((RTreeTypeAwareTupleWriter) tupleWriter);
+ frameTuple.resetByTupleIndex(this, k);
+ double LowerKey = keyValueProviders[i]
+ .getValue(frameTuple.getFieldData(i), frameTuple.getFieldStart(i));
+ double UpperKey = keyValueProviders[j]
+ .getValue(frameTuple.getFieldData(j), frameTuple.getFieldStart(j));
- int tupleOff = slotManager.getTupleOff(slotManager.getSlotEndOff());
- frameTuple.resetByTupleOffset(buf, tupleOff);
- int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0,
- keyValueProviders.length);
+ tupleEntries1.add(k, LowerKey);
+ tupleEntries2.add(k, UpperKey);
+ }
+ double LowerKey = keyValueProviders[i].getValue(tuple.getFieldData(i), tuple.getFieldStart(i));
+ double UpperKey = keyValueProviders[j].getValue(tuple.getFieldData(j), tuple.getFieldStart(j));
- splitKey.initData(splitKeySize);
- this.adjustMBR(tuples);
- rTreeTupleWriterLeftFrame.writeTupleFields(tuples, 0,
- rTreeSplitKey.getLeftPageBuffer(), 0);
- rTreeSplitKey.getLeftTuple().resetByTupleOffset(
- rTreeSplitKey.getLeftPageBuffer(), 0);
- }
+ tupleEntries1.add(-1, LowerKey);
+ tupleEntries2.add(-1, UpperKey);
- @Override
- public int getPageHeaderSize() {
- return rightPageOff;
- }
+ tupleEntries1.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
+ tupleEntries2.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
+
+ double lowerMargin = 0.0, upperMargin = 0.0;
+ // generate distribution
+ for (int k = 1; k <= splitDistribution; ++k) {
+ int d = m - 1 + k;
+
+ generateDist(tuple, tupleEntries1, rec[0], 0, d);
+ generateDist(tuple, tupleEntries2, rec[1], 0, d);
+ generateDist(tuple, tupleEntries1, rec[2], d, getTupleCount() + 1);
+ generateDist(tuple, tupleEntries2, rec[3], d, getTupleCount() + 1);
+
+ // calculate the margin of the distributions
+ lowerMargin += rec[0].margin() + rec[2].margin();
+ upperMargin += rec[1].margin() + rec[3].margin();
+ }
+ double margin = Math.min(lowerMargin, upperMargin);
+
+ // store minimum margin as split axis
+ if (margin < minMargin) {
+ minMargin = margin;
+ splitAxis = i;
+ sortOrder = (lowerMargin < upperMargin) ? 0 : 2;
+ }
+
+ tupleEntries1.clear();
+ tupleEntries2.clear();
+ }
+
+ for (int i = 0; i < getTupleCount(); ++i) {
+ frameTuple.resetByTupleIndex(this, i);
+ double key = keyValueProviders[splitAxis + sortOrder].getValue(
+ frameTuple.getFieldData(splitAxis + sortOrder), frameTuple.getFieldStart(splitAxis + sortOrder));
+ tupleEntries1.add(i, key);
+ }
+ double key = keyValueProviders[splitAxis + sortOrder].getValue(tuple.getFieldData(splitAxis + sortOrder),
+ tuple.getFieldStart(splitAxis + sortOrder));
+ tupleEntries1.add(-1, key);
+ tupleEntries1.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
+
+ double minArea = Double.MAX_VALUE;
+ double minOverlap = Double.MAX_VALUE;
+ int splitPoint = 0;
+ for (int i = 1; i <= splitDistribution; ++i) {
+ int d = m - 1 + i;
+
+ generateDist(tuple, tupleEntries1, rec[0], 0, d);
+ generateDist(tuple, tupleEntries1, rec[2], d, getTupleCount() + 1);
+
+ double overlap = rec[0].overlappedArea(rec[2]);
+ if (overlap < minOverlap) {
+ splitPoint = d;
+ minOverlap = overlap;
+ minArea = rec[0].area() + rec[2].area();
+ } else if (overlap == minOverlap) {
+ double area = rec[0].area() + rec[2].area();
+ if (area < minArea) {
+ splitPoint = d;
+ minArea = area;
+ }
+ }
+ }
+ int startIndex, endIndex;
+ if (splitPoint < (getTupleCount() + 1) / 2) {
+ startIndex = 0;
+ endIndex = splitPoint;
+ } else {
+ startIndex = splitPoint;
+ endIndex = (getTupleCount() + 1);
+ }
+ boolean tupleInserted = false;
+ int totalBytes = 0, numOfDeletedTuples = 0;
+ for (int i = startIndex; i < endIndex; i++) {
+ if (tupleEntries1.get(i).getTupleIndex() != -1) {
+ frameTuple.resetByTupleIndex(this, tupleEntries1.get(i).getTupleIndex());
+ rightFrame.insert(frameTuple, -1);
+ ((UnorderedSlotManager) slotManager).modifySlot(
+ slotManager.getSlotOff(tupleEntries1.get(i).getTupleIndex()), -1);
+ totalBytes += getTupleSize(frameTuple);
+ numOfDeletedTuples++;
+ } else {
+ rightFrame.insert(tuple, -1);
+ tupleInserted = true;
+ }
+ }
+
+ ((UnorderedSlotManager) slotManager).deleteEmptySlots();
+
+ // maintain space information
+ buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + totalBytes
+ + (slotManager.getSlotSize() * numOfDeletedTuples));
+
+ // compact both pages
+ rightFrame.compact();
+ compact();
+
+ if (!tupleInserted) {
+ insert(tuple, -1);
+ }
+
+ int tupleOff = slotManager.getTupleOff(slotManager.getSlotEndOff());
+ frameTuple.resetByTupleOffset(buf, tupleOff);
+ int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0, keyValueProviders.length);
+
+ splitKey.initData(splitKeySize);
+ this.adjustMBR();
+ rTreeTupleWriterLeftFrame.writeTupleFields(getTuples(), 0, rTreeSplitKey.getLeftPageBuffer(), 0);
+ rTreeSplitKey.getLeftTuple().resetByTupleOffset(rTreeSplitKey.getLeftPageBuffer(), 0);
+
+ ((IRTreeFrame) rightFrame).adjustMBR();
+ rTreeTupleWriterRightFrame.writeTupleFields(((RTreeNSMFrame) rightFrame).getTuples(), 0,
+ rTreeSplitKey.getRightPageBuffer(), 0);
+ rTreeSplitKey.getRightTuple().resetByTupleOffset(rTreeSplitKey.getRightPageBuffer(), 0);
+
+ tupleEntries1.clear();
+ tupleEntries2.clear();
+ }
+
+ abstract public int getTupleSize(ITupleReference tuple);
+
+ public void generateDist(ITupleReference tuple, TupleEntryArrayList entries, Rectangle rec, int start, int end) {
+ int j = 0;
+ while (entries.get(j).getTupleIndex() == -1) {
+ j++;
+ }
+ frameTuple.resetByTupleIndex(this, entries.get(j).getTupleIndex());
+ rec.set(frameTuple, keyValueProviders);
+ for (int i = start; i < end; ++i) {
+ if (i != j) {
+ if (entries.get(i).getTupleIndex() != -1) {
+ frameTuple.resetByTupleIndex(this, entries.get(i).getTupleIndex());
+ rec.enlarge(frameTuple, keyValueProviders);
+ } else {
+ rec.enlarge(tuple, keyValueProviders);
+ }
+ }
+ }
+ }
+
+ public void adjustMBRImpl(ITreeIndexTupleReference[] tuples) {
+ int maxFieldPos = keyValueProviders.length / 2;
+ for (int i = 1; i < getTupleCount(); i++) {
+ frameTuple.resetByTupleIndex(this, i);
+ for (int j = 0; j < maxFieldPos; j++) {
+ int k = maxFieldPos + j;
+ double valA = keyValueProviders[j].getValue(frameTuple.getFieldData(j), frameTuple.getFieldStart(j));
+ double valB = keyValueProviders[j].getValue(tuples[j].getFieldData(j), tuples[j].getFieldStart(j));
+ if (valA < valB) {
+ tuples[j].resetByTupleIndex(this, i);
+ }
+ valA = keyValueProviders[k].getValue(frameTuple.getFieldData(k), frameTuple.getFieldStart(k));
+ valB = keyValueProviders[k].getValue(tuples[k].getFieldData(k), tuples[k].getFieldStart(k));
+ if (valA > valB) {
+ tuples[k].resetByTupleIndex(this, i);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void adjustMBR() {
+ for (int i = 0; i < tuples.length; i++) {
+ tuples[i].setFieldCount(getFieldCount());
+ tuples[i].resetByTupleIndex(this, 0);
+ }
+
+ adjustMBRImpl(tuples);
+ }
+
+ public abstract int getFieldCount();
+
+ @Override
+ public int getPageHeaderSize() {
+ return rightPageOff;
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
index f3e6be2..63387ef 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
@@ -24,31 +24,27 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ISplitKey;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.SlotOffTupleOff;
-import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.rtree.impls.EntriesOrder;
import edu.uci.ics.hyracks.storage.am.rtree.impls.PathList;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeSplitKey;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.UnorderedSlotManager;
-import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriter;
public class RTreeNSMInteriorFrame extends RTreeNSMFrame implements IRTreeInteriorFrame {
private static final int childPtrSize = 4;
- private static IBinaryComparator childPtrCmp = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY)
+ private IBinaryComparator childPtrCmp = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY)
.createBinaryComparator();
+ private final int keyFieldCount;
public RTreeNSMInteriorFrame(ITreeIndexTupleWriter tupleWriter, IPrimitiveValueProvider[] keyValueProviders) {
super(tupleWriter, keyValueProviders);
- frameTuple.setFieldCount(keyValueProviders.length);
+ keyFieldCount = keyValueProviders.length;
+ frameTuple.setFieldCount(keyFieldCount);
}
@Override
@@ -151,6 +147,13 @@
}
@Override
+ public ITreeIndexTupleReference createTupleReference() {
+ ITreeIndexTupleReference tuple = tupleWriter.createTupleReference();
+ tuple.setFieldCount(keyFieldCount);
+ return tuple;
+ }
+
+ @Override
public int getBestChildPageId() {
return buf.getInt(getChildPointerOff(frameTuple));
}
@@ -169,6 +172,12 @@
}
@Override
+ public int getChildPageId(int tupleIndex) {
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ return buf.getInt(getChildPointerOff(frameTuple));
+ }
+
+ @Override
public int getChildPageIdIfIntersect(ITupleReference tuple, int tupleIndex, MultiComparator cmp) {
frameTuple.setFieldCount(cmp.getKeyFieldCount());
frameTuple.resetByTupleIndex(this, tupleIndex);
@@ -245,10 +254,7 @@
@Override
public FrameOpSpaceStatus hasSpaceInsert(ITupleReference tuple) {
- int bytesRequired = tupleWriter.bytesRequired(tuple) + childPtrSize; // for
- // the
- // child
- // pointer
+ int bytesRequired = tupleWriter.bytesRequired(tuple) + childPtrSize;
if (bytesRequired + slotManager.getSlotSize() <= buf.capacity() - buf.getInt(freeSpaceOff)
- (buf.getInt(tupleCountOff) * slotManager.getSlotSize()))
return FrameOpSpaceStatus.SUFFICIENT_CONTIGUOUS_SPACE;
@@ -259,14 +265,18 @@
}
@Override
- public void adjustKey(ITupleReference tuple, int tupleIndex, MultiComparator cmp) {
+ public void adjustKey(ITupleReference tuple, int tupleIndex, MultiComparator cmp) throws TreeIndexException {
frameTuple.setFieldCount(cmp.getKeyFieldCount());
if (tupleIndex == -1) {
tupleIndex = findTupleByPointer(tuple, cmp);
}
if (tupleIndex != -1) {
tupleWriter.writeTuple(tuple, buf.array(), getTupleOffset(tupleIndex));
+ } else {
+ throw new TreeIndexException("Error: Faild to find a tuple in a page");
+
}
+
}
private int pointerCmp(ITupleReference tupleA, ITupleReference tupleB, MultiComparator cmp) {
@@ -275,157 +285,8 @@
tupleB.getFieldData(cmp.getKeyFieldCount() - 1), getChildPointerOff(tupleB), childPtrSize);
}
- @Override
- public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey) throws TreeIndexException {
- RTreeSplitKey rTreeSplitKey = ((RTreeSplitKey) splitKey);
- RTreeTypeAwareTupleWriter rTreeTupleWriterLeftFrame = ((RTreeTypeAwareTupleWriter) tupleWriter);
- RTreeTypeAwareTupleWriter rTreeTupleWriterRightFrame = ((RTreeTypeAwareTupleWriter) rightFrame.getTupleWriter());
-
- // calculations are based on the R*-tree paper
- int m = (int) Math.floor((getTupleCount() + 1) * splitFactor);
- int splitDistribution = getTupleCount() - (2 * m) + 2;
-
- // to calculate the minimum margin in order to pick the split axis
- double minMargin = Double.MAX_VALUE;
- int splitAxis = 0, sortOrder = 0;
-
- int maxFieldPos = keyValueProviders.length / 2;
- for (int i = 0; i < maxFieldPos; i++) {
- int j = maxFieldPos + i;
- for (int k = 0; k < getTupleCount(); ++k) {
-
- frameTuple.resetByTupleIndex(this, k);
- double LowerKey = keyValueProviders[i]
- .getValue(frameTuple.getFieldData(i), frameTuple.getFieldStart(i));
- double UpperKey = keyValueProviders[j]
- .getValue(frameTuple.getFieldData(j), frameTuple.getFieldStart(j));
-
- tupleEntries1.add(k, LowerKey);
- tupleEntries2.add(k, UpperKey);
- }
- double LowerKey = keyValueProviders[i].getValue(tuple.getFieldData(i), tuple.getFieldStart(i));
- double UpperKey = keyValueProviders[j].getValue(tuple.getFieldData(j), tuple.getFieldStart(j));
-
- tupleEntries1.add(-1, LowerKey);
- tupleEntries2.add(-1, UpperKey);
-
- tupleEntries1.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
- tupleEntries2.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
-
- double lowerMargin = 0.0, upperMargin = 0.0;
- // generate distribution
- for (int k = 1; k <= splitDistribution; ++k) {
- int d = m - 1 + k;
-
- generateDist(tuple, tupleEntries1, rec[0], 0, d);
- generateDist(tuple, tupleEntries2, rec[1], 0, d);
- generateDist(tuple, tupleEntries1, rec[2], d, getTupleCount() + 1);
- generateDist(tuple, tupleEntries2, rec[3], d, getTupleCount() + 1);
-
- // calculate the margin of the distributions
- lowerMargin += rec[0].margin() + rec[2].margin();
- upperMargin += rec[1].margin() + rec[3].margin();
- }
- double margin = Math.min(lowerMargin, upperMargin);
-
- // store minimum margin as split axis
- if (margin < minMargin) {
- minMargin = margin;
- splitAxis = i;
- sortOrder = (lowerMargin < upperMargin) ? 0 : 2;
- }
-
- tupleEntries1.clear();
- tupleEntries2.clear();
- }
-
- for (int i = 0; i < getTupleCount(); ++i) {
- frameTuple.resetByTupleIndex(this, i);
- double key = keyValueProviders[splitAxis + sortOrder].getValue(
- frameTuple.getFieldData(splitAxis + sortOrder), frameTuple.getFieldStart(splitAxis + sortOrder));
- tupleEntries1.add(i, key);
- }
- double key = keyValueProviders[splitAxis + sortOrder].getValue(tuple.getFieldData(splitAxis + sortOrder),
- tuple.getFieldStart(splitAxis + sortOrder));
- tupleEntries1.add(-1, key);
- tupleEntries1.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
-
- double minArea = Double.MAX_VALUE;
- double minOverlap = Double.MAX_VALUE;
- int splitPoint = 0;
- for (int i = 1; i <= splitDistribution; ++i) {
- int d = m - 1 + i;
-
- generateDist(tuple, tupleEntries1, rec[0], 0, d);
- generateDist(tuple, tupleEntries1, rec[2], d, getTupleCount() + 1);
-
- double overlap = rec[0].overlappedArea(rec[2]);
- if (overlap < minOverlap) {
- splitPoint = d;
- minOverlap = overlap;
- minArea = rec[0].area() + rec[2].area();
- } else if (overlap == minOverlap) {
- double area = rec[0].area() + rec[2].area();
- if (area < minArea) {
- splitPoint = d;
- minArea = area;
- }
- }
- }
- int startIndex, endIndex;
- if (splitPoint < (getTupleCount() + 1) / 2) {
- startIndex = 0;
- endIndex = splitPoint;
- } else {
- startIndex = splitPoint;
- endIndex = (getTupleCount() + 1);
- }
- boolean tupleInserted = false;
- int totalBytes = 0, numOfDeletedTuples = 0;
- for (int i = startIndex; i < endIndex; i++) {
- if (tupleEntries1.get(i).getTupleIndex() != -1) {
- frameTuple.resetByTupleIndex(this, tupleEntries1.get(i).getTupleIndex());
- rightFrame.insert(frameTuple, -1);
- ((UnorderedSlotManager) slotManager).modifySlot(
- slotManager.getSlotOff(tupleEntries1.get(i).getTupleIndex()), -1);
- totalBytes += tupleWriter.bytesRequired(frameTuple) + childPtrSize;
- numOfDeletedTuples++;
- } else {
- rightFrame.insert(tuple, -1);
- tupleInserted = true;
- }
- }
-
- ((UnorderedSlotManager) slotManager).deleteEmptySlots();
-
- // maintain space information
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + totalBytes
- + (slotManager.getSlotSize() * numOfDeletedTuples));
-
- // compact both pages
- rightFrame.compact();
- compact();
-
- if (!tupleInserted) {
- insert(tuple, -1);
- }
-
- int tupleOff = slotManager.getTupleOff(slotManager.getSlotEndOff());
- frameTuple.resetByTupleOffset(buf, tupleOff);
- int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0, keyValueProviders.length);
-
- splitKey.initData(splitKeySize);
- this.adjustMBR(tuples);
- rTreeTupleWriterLeftFrame.writeTupleFields(tuples, 0, rTreeSplitKey.getLeftPageBuffer(), 0);
- rTreeSplitKey.getLeftTuple().resetByTupleOffset(rTreeSplitKey.getLeftPageBuffer(), 0);
-
- ((IRTreeFrame) rightFrame).adjustMBR(((RTreeNSMFrame) rightFrame).getTuples());
- rTreeTupleWriterRightFrame.writeTupleFields(((RTreeNSMFrame) rightFrame).getTuples(), 0,
- rTreeSplitKey.getRightPageBuffer(), 0);
- rTreeSplitKey.getRightTuple().resetByTupleOffset(rTreeSplitKey.getRightPageBuffer(), 0);
-
- tupleEntries1.clear();
- tupleEntries2.clear();
+ public int getTupleSize(ITupleReference tuple) {
+ return tupleWriter.bytesRequired(tuple) + childPtrSize;
}
private int getChildPointerOff(ITupleReference tuple) {
@@ -437,7 +298,7 @@
frameTuple.setFieldCount(tuple.getFieldCount());
slotManager.insertSlot(-1, buf.getInt(freeSpaceOff));
int freeSpace = buf.getInt(freeSpaceOff);
- int bytesWritten = tupleWriter.writeTupleFields(tuple, 0, tuple.getFieldCount(), buf, freeSpace);
+ int bytesWritten = tupleWriter.writeTupleFields(tuple, 0, tuple.getFieldCount(), buf.array(), freeSpace);
System.arraycopy(tuple.getFieldData(tuple.getFieldCount() - 1), getChildPointerOff(tuple), buf.array(),
freeSpace + bytesWritten, childPtrSize);
int tupleSize = bytesWritten + childPtrSize;
@@ -585,6 +446,27 @@
}
@Override
+ public boolean checkEnlargement(ITupleReference tuple, MultiComparator cmp) {
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ int c = cmp.getComparators()[i].compare(frameTuple.getFieldData(i), frameTuple.getFieldStart(i),
+ frameTuple.getFieldLength(i), tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i));
+ if (c > 0) {
+ return true;
+ }
+ c = cmp.getComparators()[j].compare(frameTuple.getFieldData(j), frameTuple.getFieldStart(j),
+ frameTuple.getFieldLength(j), tuple.getFieldData(j), tuple.getFieldStart(j),
+ tuple.getFieldLength(j));
+ if (c < 0) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
public void enlarge(ITupleReference tuple, MultiComparator cmp) {
int maxFieldPos = cmp.getKeyFieldCount() / 2;
for (int i = 0; i < maxFieldPos; i++) {
@@ -606,13 +488,25 @@
}
}
- @Override
- public void adjustMBR(ITreeIndexTupleReference[] tuples) {
- for (int i = 0; i < tuples.length; i++) {
- tuples[i].setFieldCount(keyValueProviders.length);
- tuples[i].resetByTupleIndex(this, 0);
+ // For debugging.
+ public ArrayList<Integer> getChildren(MultiComparator cmp) {
+ ArrayList<Integer> ret = new ArrayList<Integer>();
+ frameTuple.setFieldCount(cmp.getKeyFieldCount());
+ int tupleCount = buf.getInt(tupleCountOff);
+ for (int i = 0; i < tupleCount; i++) {
+ int tupleOff = slotManager.getTupleOff(slotManager.getSlotOff(i));
+ frameTuple.resetByTupleOffset(buf, tupleOff);
+ int intVal = IntegerSerializerDeserializer.getInt(
+ buf.array(),
+ frameTuple.getFieldStart(frameTuple.getFieldCount() - 1)
+ + frameTuple.getFieldLength(frameTuple.getFieldCount() - 1));
+ ret.add(intVal);
}
+ return ret;
+ }
- adjustMBRImpl(tuples);
+ @Override
+ public int getFieldCount() {
+ return keyValueProviders.length;
}
}
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrame.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrame.java
index b2ae5fc..f1d71ff 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrame.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrame.java
@@ -17,18 +17,10 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ISplitKey;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
-import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.EntriesOrder;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeSplitKey;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.UnorderedSlotManager;
-import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriter;
public class RTreeNSMLeafFrame extends RTreeNSMFrame implements IRTreeLeafFrame {
@@ -37,6 +29,11 @@
}
@Override
+ public ITreeIndexTupleReference createTupleReference() {
+ return tupleWriter.createTupleReference();
+ }
+
+ @Override
public int findTupleIndex(ITupleReference tuple, MultiComparator cmp) {
return slotManager.findTupleIndex(tuple, frameTuple, cmp, null, null);
}
@@ -63,159 +60,8 @@
return true;
}
- @Override
- public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey) throws TreeIndexException {
-
- RTreeSplitKey rTreeSplitKey = ((RTreeSplitKey) splitKey);
- RTreeTypeAwareTupleWriter rTreeTupleWriterLeftFrame = ((RTreeTypeAwareTupleWriter) tupleWriter);
- RTreeTypeAwareTupleWriter rTreeTupleWriterRightFrame = ((RTreeTypeAwareTupleWriter) rightFrame.getTupleWriter());
-
- // calculations are based on the R*-tree paper
- int m = (int) Math.floor((getTupleCount() + 1) * splitFactor);
- int splitDistribution = getTupleCount() - (2 * m) + 2;
-
- // to calculate the minimum margin in order to pick the split axis
- double minMargin = Double.MAX_VALUE;
- int splitAxis = 0, sortOrder = 0;
-
- int maxFieldPos = keyValueProviders.length / 2;
- for (int i = 0; i < maxFieldPos; i++) {
- int j = maxFieldPos + i;
- for (int k = 0; k < getTupleCount(); ++k) {
-
- frameTuple.resetByTupleIndex(this, k);
-
- double LowerKey = keyValueProviders[i].getValue(frameTuple.getFieldData(i),
- frameTuple.getFieldStart(i));
- double UpperKey = keyValueProviders[j].getValue(frameTuple.getFieldData(j),
- frameTuple.getFieldStart(j));
-
- tupleEntries1.add(k, LowerKey);
- tupleEntries2.add(k, UpperKey);
- }
- double LowerKey = keyValueProviders[i].getValue(tuple.getFieldData(i), tuple.getFieldStart(i));
- double UpperKey = keyValueProviders[j].getValue(tuple.getFieldData(j), tuple.getFieldStart(j));
-
- tupleEntries1.add(-1, LowerKey);
- tupleEntries2.add(-1, UpperKey);
-
- tupleEntries1.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
- tupleEntries2.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
-
- double lowerMargin = 0.0, upperMargin = 0.0;
- // generate distribution
- for (int k = 1; k <= splitDistribution; ++k) {
- int d = m - 1 + k;
-
- generateDist(tuple, tupleEntries1, rec[0], 0, d);
- generateDist(tuple, tupleEntries2, rec[1], 0, d);
- generateDist(tuple, tupleEntries1, rec[2], d, getTupleCount() + 1);
- generateDist(tuple, tupleEntries2, rec[3], d, getTupleCount() + 1);
-
- // calculate the margin of the distributions
- lowerMargin += rec[0].margin() + rec[2].margin();
- upperMargin += rec[1].margin() + rec[3].margin();
- }
- double margin = Math.min(lowerMargin, upperMargin);
-
- // store minimum margin as split axis
- if (margin < minMargin) {
- minMargin = margin;
- splitAxis = i;
- sortOrder = (lowerMargin < upperMargin) ? 0 : 2;
- }
-
- tupleEntries1.clear();
- tupleEntries2.clear();
- }
-
- for (int i = 0; i < getTupleCount(); ++i) {
- frameTuple.resetByTupleIndex(this, i);
- double key = keyValueProviders[splitAxis + sortOrder].getValue(
- frameTuple.getFieldData(splitAxis + sortOrder), frameTuple.getFieldStart(splitAxis + sortOrder));
- tupleEntries1.add(i, key);
- }
- double key = keyValueProviders[splitAxis + sortOrder].getValue(tuple.getFieldData(splitAxis + sortOrder),
- tuple.getFieldStart(splitAxis + sortOrder));
- tupleEntries1.add(-1, key);
- tupleEntries1.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
-
- double minArea = Double.MAX_VALUE;
- double minOverlap = Double.MAX_VALUE;
- int splitPoint = 0;
- for (int i = 1; i <= splitDistribution; ++i) {
- int d = m - 1 + i;
-
- generateDist(tuple, tupleEntries1, rec[0], 0, d);
- generateDist(tuple, tupleEntries1, rec[2], d, getTupleCount() + 1);
-
- double overlap = rec[0].overlappedArea(rec[2]);
- if (overlap < minOverlap) {
- splitPoint = d;
- minOverlap = overlap;
- minArea = rec[0].area() + rec[2].area();
- } else if (overlap == minOverlap) {
- double area = rec[0].area() + rec[2].area();
- if (area < minArea) {
- splitPoint = d;
- minArea = area;
- }
- }
- }
- int startIndex, endIndex;
- if (splitPoint < (getTupleCount() + 1) / 2) {
- startIndex = 0;
- endIndex = splitPoint;
- } else {
- startIndex = splitPoint;
- endIndex = (getTupleCount() + 1);
- }
- boolean tupleInserted = false;
- int totalBytes = 0, numOfDeletedTuples = 0;
- for (int i = startIndex; i < endIndex; i++) {
- if (tupleEntries1.get(i).getTupleIndex() != -1) {
- frameTuple.resetByTupleIndex(this, tupleEntries1.get(i).getTupleIndex());
- rightFrame.insert(frameTuple, -1);
- ((UnorderedSlotManager) slotManager).modifySlot(
- slotManager.getSlotOff(tupleEntries1.get(i).getTupleIndex()), -1);
- totalBytes += tupleWriter.bytesRequired(frameTuple);
- numOfDeletedTuples++;
- } else {
- rightFrame.insert(tuple, -1);
- tupleInserted = true;
- }
- }
-
- ((UnorderedSlotManager) slotManager).deleteEmptySlots();
-
- // maintain space information
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + totalBytes
- + (slotManager.getSlotSize() * numOfDeletedTuples));
-
- // compact both pages
- rightFrame.compact();
- compact();
-
- if (!tupleInserted) {
- insert(tuple, -1);
- }
-
- int tupleOff = slotManager.getTupleOff(slotManager.getSlotEndOff());
- frameTuple.resetByTupleOffset(buf, tupleOff);
- int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0, keyValueProviders.length);
-
- splitKey.initData(splitKeySize);
- this.adjustMBR(tuples);
- rTreeTupleWriterLeftFrame.writeTupleFields(tuples, 0, rTreeSplitKey.getLeftPageBuffer(), 0);
- rTreeSplitKey.getLeftTuple().resetByTupleOffset(rTreeSplitKey.getLeftPageBuffer(), 0);
-
- ((IRTreeFrame) rightFrame).adjustMBR(((RTreeNSMFrame) rightFrame).getTuples());
- rTreeTupleWriterRightFrame.writeTupleFields(((RTreeNSMFrame) rightFrame).getTuples(), 0,
- rTreeSplitKey.getRightPageBuffer(), 0);
- rTreeSplitKey.getRightTuple().resetByTupleOffset(rTreeSplitKey.getRightPageBuffer(), 0);
-
- tupleEntries1.clear();
- tupleEntries2.clear();
+ public int getTupleSize(ITupleReference tuple) {
+ return tupleWriter.bytesRequired(tuple);
}
@Override
@@ -247,10 +93,7 @@
}
@Override
- public void adjustMBR(ITreeIndexTupleReference[] tuples) {
- for (int i = 0; i < tuples.length; i++) {
- tuples[i].resetByTupleIndex(this, 0);
- }
- adjustMBRImpl(tuples);
+ public int getFieldCount() {
+ return frameTuple.getFieldCount();
}
}
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/PathList.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/PathList.java
index 4f86111..911029d 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/PathList.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/PathList.java
@@ -19,93 +19,99 @@
import edu.uci.ics.hyracks.storage.am.common.ophelpers.LongArrayList;
public class PathList {
- private IntArrayList pageIds;
- private LongArrayList pageLsns;
- private IntArrayList pageIndexes;
+ private IntArrayList pageIds;
+ private LongArrayList pageLsns;
+ private IntArrayList pageIndexes;
- public PathList(int initialCapacity, int growth) {
- pageIds = new IntArrayList(initialCapacity, growth);
- pageLsns = new LongArrayList(initialCapacity, growth);
- pageIndexes = new IntArrayList(initialCapacity, growth);
- }
+ public PathList(int initialCapacity, int growth) {
+ pageIds = new IntArrayList(initialCapacity, growth);
+ pageLsns = new LongArrayList(initialCapacity, growth);
+ pageIndexes = new IntArrayList(initialCapacity, growth);
+ }
- public int size() {
- return pageIds.size();
- }
+ public int size() {
+ return pageIds.size();
+ }
- public int first() {
- return pageIds.first();
- }
+ public int first() {
+ return pageIds.first();
+ }
- public void add(int pageId, long pageLsn, int pageIndex) {
- pageIds.add(pageId);
- pageLsns.add(pageLsn);
- pageIndexes.add(pageIndex);
- }
+ public void add(int pageId, long pageLsn, int pageIndex) {
+ pageIds.add(pageId);
+ pageLsns.add(pageLsn);
+ pageIndexes.add(pageIndex);
+ }
- public int getFirstPageId() {
- return pageIds.getFirst();
- }
+ public void addFirst(int pageId, long pageLsn, int pageIndex) {
+ pageIds.addFirst(pageId);
+ pageLsns.addFirst(pageLsn);
+ pageIndexes.addFirst(pageIndex);
+ }
- public long getFirstPageLsn() {
- return pageLsns.getFirst();
- }
+ public int getFirstPageId() {
+ return pageIds.getFirst();
+ }
- public int getFirstPageIndex() {
- return pageIndexes.getFirst();
- }
+ public long getFirstPageLsn() {
+ return pageLsns.getFirst();
+ }
- public int getLastPageId() {
- return pageIds.getLast();
- }
+ public int getFirstPageIndex() {
+ return pageIndexes.getFirst();
+ }
- public long getLastPageLsn() {
- return pageLsns.getLast();
- }
+ public int getLastPageId() {
+ return pageIds.getLast();
+ }
- public int getLastPageIndex() {
- return pageIndexes.getLast();
- }
+ public long getLastPageLsn() {
+ return pageLsns.getLast();
+ }
- public int getPageId(int i) {
- return pageIds.get(i);
- }
+ public int getLastPageIndex() {
+ return pageIndexes.getLast();
+ }
- public long getPageLsn(int i) {
- return pageLsns.get(i);
- }
+ public int getPageId(int i) {
+ return pageIds.get(i);
+ }
- public int getPageIndex(int i) {
- return pageIndexes.get(i);
- }
+ public long getPageLsn(int i) {
+ return pageLsns.get(i);
+ }
- public void setPageLsn(int i, long pageLsn) {
- pageLsns.set(i, pageLsn);
- }
+ public int getPageIndex(int i) {
+ return pageIndexes.get(i);
+ }
- public void moveFirst() {
- pageIds.moveFirst();
- pageLsns.moveFirst();
- pageIndexes.moveFirst();
- }
+ public void setPageLsn(int i, long pageLsn) {
+ pageLsns.set(i, pageLsn);
+ }
- public void moveLast() {
- pageIds.removeLast();
- pageLsns.removeLast();
- pageIndexes.removeLast();
- }
+ public void moveFirst() {
+ pageIds.moveFirst();
+ pageLsns.moveFirst();
+ pageIndexes.moveFirst();
+ }
- public boolean isLast() {
- return pageIds.isLast();
- }
+ public void moveLast() {
+ pageIds.removeLast();
+ pageLsns.removeLast();
+ pageIndexes.removeLast();
+ }
- public void clear() {
- pageIds.clear();
- pageLsns.clear();
- pageIndexes.clear();
- }
+ public boolean isLast() {
+ return pageIds.isLast();
+ }
- public boolean isEmpty() {
- return pageIds.isEmpty();
- }
+ public void clear() {
+ pageIds.clear();
+ pageLsns.clear();
+ pageIndexes.clear();
+ }
+
+ public boolean isEmpty() {
+ return pageIds.isEmpty();
+ }
}
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
index 5f78dcc..cc3cf5b 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
@@ -20,11 +20,13 @@
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
@@ -33,8 +35,8 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
-import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
@@ -44,161 +46,119 @@
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrame;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
public class RTree implements ITreeIndex {
- private boolean created = false;
- private boolean loaded = false;
- private final int rootPage = 1; // the root page never changes
+ private final int rootPage = 1;
- private final AtomicLong globalNsn; // Global node sequence number
- private int numOfPages = 1;
+ // Global node sequence number used for the concurrency control protocol
+ private final AtomicLong globalNsn;
private final ReadWriteLock treeLatch;
private final IFreePageManager freePageManager;
private final IBufferCache bufferCache;
private int fileId;
- private final SearchPredicate diskOrderScanPredicate;
private final ITreeIndexFrameFactory interiorFrameFactory;
private final ITreeIndexFrameFactory leafFrameFactory;
private final int fieldCount;
- private final MultiComparator cmp;
+ private final IBinaryComparatorFactory[] cmpFactories;
- public int rootSplits = 0;
- public int[] splitsByLevel = new int[500];
- public AtomicLong readLatchesAcquired = new AtomicLong();
- public AtomicLong readLatchesReleased = new AtomicLong();
- public AtomicLong writeLatchesAcquired = new AtomicLong();
- public AtomicLong writeLatchesReleased = new AtomicLong();
- public AtomicLong pins = new AtomicLong();
- public AtomicLong unpins = new AtomicLong();
- public byte currentLevel = 0;
-
- // TODO: is MultiComparator needed at all?
- public RTree(IBufferCache bufferCache, int fieldCount, MultiComparator cmp, IFreePageManager freePageManager,
- ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
+ public RTree(IBufferCache bufferCache, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
+ IFreePageManager freePageManager, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory leafFrameFactory) {
this.bufferCache = bufferCache;
this.fieldCount = fieldCount;
- this.cmp = cmp;
+ this.cmpFactories = cmpFactories;
this.freePageManager = freePageManager;
this.interiorFrameFactory = interiorFrameFactory;
this.leafFrameFactory = leafFrameFactory;
globalNsn = new AtomicLong();
this.treeLatch = new ReentrantReadWriteLock(true);
- this.diskOrderScanPredicate = new SearchPredicate(null, cmp);
}
- public void incrementGlobalNsn() {
- globalNsn.incrementAndGet();
+ private long incrementGlobalNsn() {
+ return globalNsn.incrementAndGet();
}
- public long getGlobalNsn() {
- return globalNsn.get();
- }
-
- public void incrementReadLatchesAcquired() {
- readLatchesAcquired.incrementAndGet();
- }
-
- public void incrementReadLatchesReleased() {
- readLatchesReleased.incrementAndGet();
- }
-
- public void incrementWriteLatchesAcquired() {
- writeLatchesAcquired.incrementAndGet();
- }
-
- public void incrementWriteLatchesReleased() {
- writeLatchesReleased.incrementAndGet();
- }
-
- public void incrementPins() {
- pins.incrementAndGet();
- }
-
- public void incrementUnpins() {
- unpins.incrementAndGet();
- }
-
- public String printStats() {
- StringBuilder strBuilder = new StringBuilder();
- strBuilder.append("\n");
- strBuilder.append("ROOTSPLITS: " + rootSplits + "\n");
- strBuilder.append("SPLITS BY LEVEL\n");
- for (int i = 0; i < currentLevel; i++) {
- strBuilder.append(String.format("%3d ", i) + String.format("%8d ", splitsByLevel[i]) + "\n");
+ public byte getTreeHeight(IRTreeLeafFrame leafFrame) throws HyracksDataException {
+ ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
+ rootNode.acquireReadLatch();
+ try {
+ leafFrame.setPage(rootNode);
+ return leafFrame.getLevel();
+ } finally {
+ rootNode.releaseReadLatch();
+ bufferCache.unpin(rootNode);
}
- strBuilder.append(String.format("READ LATCHES: %10d %10d\n", readLatchesAcquired.get(),
- readLatchesReleased.get()));
- strBuilder.append(String.format("WRITE LATCHES: %10d %10d\n", writeLatchesAcquired.get(),
- writeLatchesReleased.get()));
- strBuilder.append(String.format("PINS: %10d %10d\n", pins.get(), unpins.get()));
+ }
- strBuilder.append(String.format("Num of Pages: %10d\n", numOfPages));
-
+ @SuppressWarnings("rawtypes")
+ public String printTree(IRTreeLeafFrame leafFrame, IRTreeInteriorFrame interiorFrame,
+ ISerializerDeserializer[] keySerdes) throws Exception {
+ MultiComparator cmp = MultiComparator.create(cmpFactories);
+ byte treeHeight = getTreeHeight(leafFrame);
+ StringBuilder strBuilder = new StringBuilder();
+ printTree(rootPage, null, false, leafFrame, interiorFrame, treeHeight, keySerdes, strBuilder, cmp);
return strBuilder.toString();
}
- public void printTree(IRTreeFrame leafFrame, IRTreeFrame interiorFrame, ISerializerDeserializer[] keySerdes)
- throws Exception {
- printTree(rootPage, null, false, leafFrame, interiorFrame, keySerdes);
- }
-
- public void printTree(int pageId, ICachedPage parent, boolean unpin, IRTreeFrame leafFrame,
- IRTreeFrame interiorFrame, ISerializerDeserializer[] keySerdes) throws Exception {
-
+ @SuppressWarnings("rawtypes")
+ public void printTree(int pageId, ICachedPage parent, boolean unpin, IRTreeLeafFrame leafFrame,
+ IRTreeInteriorFrame interiorFrame, byte treeHeight, ISerializerDeserializer[] keySerdes,
+ StringBuilder strBuilder, MultiComparator cmp) throws Exception {
ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- incrementPins();
node.acquireReadLatch();
- incrementReadLatchesAcquired();
-
try {
if (parent != null && unpin == true) {
parent.releaseReadLatch();
- incrementReadLatchesReleased();
bufferCache.unpin(parent);
- incrementUnpins();
}
-
interiorFrame.setPage(node);
int level = interiorFrame.getLevel();
-
- System.out.format("%1d ", level);
- System.out.format("%3d ", pageId);
- for (int i = 0; i < currentLevel - level; i++)
- System.out.format(" ");
+ strBuilder.append(String.format("%1d ", level));
+ strBuilder.append(String.format("%3d ", pageId) + ": ");
+ for (int i = 0; i < treeHeight - level; i++) {
+ strBuilder.append(" ");
+ }
String keyString;
+ long LSN, NSN;
+ int rightPage;
if (interiorFrame.isLeaf()) {
leafFrame.setPage(node);
keyString = TreeIndexUtils.printFrameTuples(leafFrame, keySerdes);
+ LSN = leafFrame.getPageLsn();
+ NSN = leafFrame.getPageNsn();
+ rightPage = leafFrame.getRightPage();
+
} else {
keyString = TreeIndexUtils.printFrameTuples(interiorFrame, keySerdes);
+ LSN = interiorFrame.getPageLsn();
+ NSN = interiorFrame.getPageNsn();
+ rightPage = interiorFrame.getRightPage();
}
- System.out.format(keyString);
+ strBuilder.append(keyString + "\n" + "pageId: " + pageId + " LSN: " + LSN + " NSN: " + NSN + " rightPage: "
+ + rightPage + "\n");
if (!interiorFrame.isLeaf()) {
- ArrayList<Integer> children = ((RTreeNSMFrame) (interiorFrame)).getChildren(cmp);
+ ArrayList<Integer> children = ((RTreeNSMInteriorFrame) (interiorFrame)).getChildren(cmp);
for (int i = 0; i < children.size(); i++) {
- printTree(children.get(i), node, i == children.size() - 1, leafFrame, interiorFrame, keySerdes);
+ printTree(children.get(i), node, i == children.size() - 1, leafFrame, interiorFrame, treeHeight,
+ keySerdes, strBuilder, cmp);
}
} else {
node.releaseReadLatch();
- incrementReadLatchesReleased();
bufferCache.unpin(node);
- incrementUnpins();
}
} catch (Exception e) {
node.releaseReadLatch();
- incrementReadLatchesReleased();
bufferCache.unpin(node);
- incrementUnpins();
- throw e;
+ e.printStackTrace();
}
}
@@ -206,94 +166,108 @@
public void create(int fileId) throws HyracksDataException {
treeLatch.writeLock().lock();
try {
- if (created) {
- return;
- }
-
ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
+ freePageManager.open(fileId);
freePageManager.init(metaFrame, rootPage);
// initialize root page
ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true);
- incrementPins();
rootNode.acquireWriteLatch();
- incrementWriteLatchesAcquired();
try {
leafFrame.setPage(rootNode);
leafFrame.initBuffer((byte) 0);
} finally {
rootNode.releaseWriteLatch();
- incrementWriteLatchesReleased();
bufferCache.unpin(rootNode);
- incrementUnpins();
}
- currentLevel = 0;
-
- created = true;
} finally {
treeLatch.writeLock().unlock();
}
}
+ @Override
public void open(int fileId) {
this.fileId = fileId;
+ freePageManager.open(fileId);
}
+ @Override
public void close() {
fileId = -1;
+ freePageManager.close();
+ }
+
+ @Override
+ public int getFileId() {
+ return fileId;
+ }
+
+ @Override
+ public IBufferCache getBufferCache() {
+ return bufferCache;
}
private RTreeOpContext createOpContext() {
return new RTreeOpContext((IRTreeLeafFrame) leafFrameFactory.createFrame(),
(IRTreeInteriorFrame) interiorFrameFactory.createFrame(), freePageManager.getMetaDataFrameFactory()
- .createFrame(), 8);
+ .createFrame(), cmpFactories, 8);
}
- private void insert(ITupleReference tuple, IIndexOpContext ictx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void insert(ITupleReference tuple, IIndexOpContext ictx) throws HyracksDataException, TreeIndexException {
RTreeOpContext ctx = (RTreeOpContext) ictx;
ctx.reset();
ctx.setTuple(tuple);
ctx.splitKey.reset();
- ctx.splitKey.getLeftTuple().setFieldCount(cmp.getKeyFieldCount());
- ctx.splitKey.getRightTuple().setFieldCount(cmp.getKeyFieldCount());
+ ctx.splitKey.getLeftTuple().setFieldCount(cmpFactories.length);
+ ctx.splitKey.getRightTuple().setFieldCount(cmpFactories.length);
- int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ int maxFieldPos = cmpFactories.length / 2;
for (int i = 0; i < maxFieldPos; i++) {
int j = maxFieldPos + i;
- int c = cmp.getComparators()[i].compare(tuple.getFieldData(i), tuple.getFieldStart(i),
+ int c = ctx.cmp.getComparators()[i].compare(tuple.getFieldData(i), tuple.getFieldStart(i),
tuple.getFieldLength(i), tuple.getFieldData(j), tuple.getFieldStart(j), tuple.getFieldLength(j));
if (c > 0) {
throw new IllegalArgumentException("The low key point has larger coordinates than the high key point.");
}
}
+ try {
+ ICachedPage leafNode = findLeaf(ctx);
- ICachedPage leafNode = findLeaf(ctx);
+ int pageId = ctx.pathList.getLastPageId();
+ ctx.pathList.moveLast();
+ insertTuple(leafNode, pageId, ctx.getTuple(), ctx, true);
- int pageId = ctx.pathList.getLastPageId();
- ctx.pathList.moveLast();
- insertTuple(leafNode, pageId, ctx.getTuple(), ctx, true);
+ while (true) {
+ if (ctx.splitKey.getLeftPageBuffer() != null) {
+ updateParentForInsert(ctx);
+ } else {
+ break;
+ }
+ }
+ } finally {
+ for (int i = ctx.NSNUpdates.size() - 1; i >= 0; i--) {
+ ICachedPage node = ctx.NSNUpdates.get(i);
+ ctx.interiorFrame.setPage(node);
+ ctx.interiorFrame.setPageNsn(incrementGlobalNsn());
+ }
- while (true) {
- if (ctx.splitKey.getLeftPageBuffer() != null) {
- updateParentForInsert(ctx);
- } else {
- break;
+ for (int i = ctx.LSNUpdates.size() - 1; i >= 0; i--) {
+ ICachedPage node = ctx.LSNUpdates.get(i);
+ ctx.interiorFrame.setPage(node);
+ ctx.interiorFrame.setPageLsn(incrementGlobalNsn());
+ node.releaseWriteLatch();
+ bufferCache.unpin(node);
}
}
-
- leafNode.releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(leafNode);
- incrementUnpins();
}
- public ICachedPage findLeaf(RTreeOpContext ctx) throws HyracksDataException {
+ private ICachedPage findLeaf(RTreeOpContext ctx) throws HyracksDataException {
int pageId = rootPage;
boolean writeLatched = false;
boolean readLatched = false;
- boolean succeed = false;
+ boolean succeeded = false;
ICachedPage node = null;
boolean isLeaf = false;
long pageLsn = 0, parentLsn = 0;
@@ -303,49 +277,38 @@
while (true) {
if (!writeLatched) {
node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- incrementPins();
ctx.interiorFrame.setPage(node);
isLeaf = ctx.interiorFrame.isLeaf();
if (isLeaf) {
node.acquireWriteLatch();
writeLatched = true;
- incrementWriteLatchesAcquired();
if (!ctx.interiorFrame.isLeaf()) {
node.releaseWriteLatch();
writeLatched = false;
- incrementWriteLatchesReleased();
bufferCache.unpin(node);
- incrementUnpins();
continue;
}
} else {
// Be optimistic and grab read latch first. We will swap
- // it
- // to write latch if we need to enlarge the best child
- // tuple.
+ // it to write latch if we need to enlarge the best
+ // child tuple.
node.acquireReadLatch();
readLatched = true;
- incrementReadLatchesAcquired();
}
}
if (pageId != rootPage && parentLsn < ctx.interiorFrame.getPageNsn()) {
// Concurrent split detected, go back to parent and
- // re-choose
- // the best child
+ // re-choose the best child
if (writeLatched) {
node.releaseWriteLatch();
writeLatched = false;
- incrementWriteLatchesReleased();
bufferCache.unpin(node);
- incrementUnpins();
} else {
node.releaseReadLatch();
readLatched = false;
- incrementReadLatchesReleased();
bufferCache.unpin(node);
- incrementUnpins();
}
pageId = ctx.pathList.getLastPageId();
@@ -361,72 +324,73 @@
if (!isLeaf) {
// findBestChild must be called *before* getBestChildPageId
- ctx.interiorFrame.findBestChild(ctx.getTuple(), cmp);
+ boolean enlarementIsNeeded = ctx.interiorFrame.findBestChild(ctx.getTuple(), ctx.cmp);
int childPageId = ctx.interiorFrame.getBestChildPageId();
- if (!writeLatched) {
- node.releaseReadLatch();
- readLatched = false;
- incrementReadLatchesReleased();
- // TODO: do we need to un-pin and pin again?
+ if (enlarementIsNeeded) {
+ if (!writeLatched) {
+ node.releaseReadLatch();
+ readLatched = false;
+ bufferCache.unpin(node);
+
+ node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ node.acquireWriteLatch();
+ writeLatched = true;
+ ctx.interiorFrame.setPage(node);
+
+ if (ctx.interiorFrame.getPageLsn() != pageLsn) {
+ // The page was changed while we unlocked it;
+ // thus, retry (re-choose best child)
+
+ ctx.pathList.moveLast();
+ continue;
+ }
+ }
+ // We don't need to reset the frameTuple because it is
+ // already pointing to the best child
+ ctx.interiorFrame.enlarge(ctx.getTuple(), ctx.cmp);
+
+ node.releaseWriteLatch();
+ writeLatched = false;
bufferCache.unpin(node);
- incrementUnpins();
-
- node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- incrementPins();
- node.acquireWriteLatch();
- writeLatched = true;
- incrementWriteLatchesAcquired();
- ctx.interiorFrame.setPage(node);
-
- if (ctx.interiorFrame.getPageLsn() != pageLsn) {
- // The page was changed while we unlocked it; thus,
- // retry (re-choose best child)
-
- ctx.pathList.moveLast();
- continue;
+ } else {
+ if (readLatched) {
+ node.releaseReadLatch();
+ readLatched = false;
+ bufferCache.unpin(node);
+ } else if (writeLatched) {
+ node.releaseWriteLatch();
+ writeLatched = false;
+ bufferCache.unpin(node);
}
}
- // We don't need to reset the frameTuple because it is
- // already pointing to the best child
- ctx.interiorFrame.enlarge(ctx.getTuple(), cmp);
-
- node.releaseWriteLatch();
- writeLatched = false;
- incrementWriteLatchesReleased();
- bufferCache.unpin(node);
- incrementUnpins();
-
pageId = childPageId;
parentLsn = pageLsn;
} else {
ctx.leafFrame.setPage(node);
- succeed = true;
+ succeeded = true;
return node;
}
}
} finally {
- if (!succeed) {
+ if (!succeeded) {
if (readLatched) {
node.releaseReadLatch();
readLatched = false;
- incrementReadLatchesReleased();
bufferCache.unpin(node);
- incrementUnpins();
} else if (writeLatched) {
node.releaseWriteLatch();
writeLatched = false;
- incrementWriteLatchesReleased();
bufferCache.unpin(node);
- incrementUnpins();
}
}
}
}
private void insertTuple(ICachedPage node, int pageId, ITupleReference tuple, RTreeOpContext ctx, boolean isLeaf)
- throws HyracksDataException, TreeIndexException, PageAllocationException {
+ throws HyracksDataException, TreeIndexException {
+ boolean succeeded = false;
FrameOpSpaceStatus spaceStatus;
if (!isLeaf) {
spaceStatus = ctx.interiorFrame.hasSpaceInsert(tuple);
@@ -436,139 +400,162 @@
switch (spaceStatus) {
case SUFFICIENT_CONTIGUOUS_SPACE: {
- if (!isLeaf) {
- ctx.interiorFrame.insert(tuple, -1);
- incrementGlobalNsn();
- ctx.interiorFrame.setPageLsn(getGlobalNsn());
- } else {
- ctx.leafFrame.insert(tuple, -1);
- incrementGlobalNsn();
- ctx.leafFrame.setPageLsn(getGlobalNsn());
+ try {
+ if (!isLeaf) {
+ ctx.interiorFrame.insert(tuple, -1);
+ } else {
+ ctx.leafFrame.insert(tuple, -1);
+ }
+ succeeded = true;
+ } finally {
+ if (succeeded) {
+ ctx.LSNUpdates.add(node);
+ ctx.splitKey.reset();
+ } else if (isLeaf) {
+ // In case of a crash, we un-latch the interior node
+ // inside updateParentForInsert.
+ node.releaseWriteLatch();
+ bufferCache.unpin(node);
+ }
}
- ctx.splitKey.reset();
break;
}
case SUFFICIENT_SPACE: {
- if (!isLeaf) {
- ctx.interiorFrame.compact();
- ctx.interiorFrame.insert(tuple, -1);
- incrementGlobalNsn();
- ctx.interiorFrame.setPageLsn(getGlobalNsn());
- } else {
- ctx.leafFrame.compact();
- ctx.leafFrame.insert(tuple, -1);
- incrementGlobalNsn();
- ctx.leafFrame.setPageLsn(getGlobalNsn());
+ try {
+ if (!isLeaf) {
+ ctx.interiorFrame.compact();
+ ctx.interiorFrame.insert(tuple, -1);
+ } else {
+ ctx.leafFrame.compact();
+ ctx.leafFrame.insert(tuple, -1);
+ }
+ succeeded = true;
+ } finally {
+ if (succeeded) {
+ ctx.LSNUpdates.add(node);
+ ctx.splitKey.reset();
+ } else if (isLeaf) {
+ // In case of a crash, we un-latch the interior node
+ // inside updateParentForInsert.
+ node.releaseWriteLatch();
+ bufferCache.unpin(node);
+ }
}
- ctx.splitKey.reset();
break;
}
case INSUFFICIENT_SPACE: {
int rightPageId = freePageManager.getFreePage(ctx.metaFrame);
ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true);
- incrementPins();
rightNode.acquireWriteLatch();
- incrementWriteLatchesAcquired();
try {
IRTreeFrame rightFrame;
- numOfPages++; // debug
if (!isLeaf) {
- splitsByLevel[ctx.interiorFrame.getLevel()]++; // debug
rightFrame = (IRTreeFrame) interiorFrameFactory.createFrame();
rightFrame.setPage(rightNode);
rightFrame.initBuffer((byte) ctx.interiorFrame.getLevel());
+ rightFrame.setRightPage(ctx.interiorFrame.getRightPage());
ctx.interiorFrame.split(rightFrame, tuple, ctx.splitKey);
ctx.interiorFrame.setRightPage(rightPageId);
- rightFrame.setPageNsn(ctx.interiorFrame.getPageNsn());
- incrementGlobalNsn();
- long newNsn = getGlobalNsn();
- rightFrame.setPageLsn(newNsn);
- ctx.interiorFrame.setPageNsn(newNsn);
- ctx.interiorFrame.setPageLsn(newNsn);
} else {
- splitsByLevel[0]++; // debug
rightFrame = (IRTreeFrame) leafFrameFactory.createFrame();
rightFrame.setPage(rightNode);
rightFrame.initBuffer((byte) 0);
+ rightFrame.setRightPage(ctx.interiorFrame.getRightPage());
ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey);
ctx.leafFrame.setRightPage(rightPageId);
- rightFrame.setPageNsn(ctx.leafFrame.getPageNsn());
- incrementGlobalNsn();
- long newNsn = getGlobalNsn();
- rightFrame.setPageLsn(newNsn);
- ctx.leafFrame.setPageNsn(newNsn);
- ctx.leafFrame.setPageLsn(newNsn);
}
- ctx.splitKey.setPages(pageId, rightPageId);
- if (pageId == rootPage) {
- rootSplits++; // debug
- splitsByLevel[currentLevel]++;
- currentLevel++;
-
- int newLeftId = freePageManager.getFreePage(ctx.metaFrame);
- ICachedPage newLeftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newLeftId),
- true);
- incrementPins();
- newLeftNode.acquireWriteLatch();
- incrementWriteLatchesAcquired();
- try {
- // copy left child to new left child
- System.arraycopy(node.getBuffer().array(), 0, newLeftNode.getBuffer().array(), 0,
- newLeftNode.getBuffer().capacity());
-
- // initialize new root (leftNode becomes new root)
- ctx.interiorFrame.setPage(node);
- ctx.interiorFrame.initBuffer((byte) (ctx.interiorFrame.getLevel() + 1));
-
- ctx.splitKey.setLeftPage(newLeftId);
-
- ctx.interiorFrame.insert(ctx.splitKey.getLeftTuple(), -1);
- ctx.interiorFrame.insert(ctx.splitKey.getRightTuple(), -1);
-
- incrementGlobalNsn();
- long newNsn = getGlobalNsn();
- ctx.interiorFrame.setPageLsn(newNsn);
- ctx.interiorFrame.setPageNsn(newNsn);
- } finally {
- newLeftNode.releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(newLeftNode);
- incrementUnpins();
- }
-
- ctx.splitKey.reset();
- }
+ succeeded = true;
} finally {
- rightNode.releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(rightNode);
- incrementUnpins();
+ if (succeeded) {
+ ctx.NSNUpdates.add(rightNode);
+ ctx.LSNUpdates.add(rightNode);
+ ctx.NSNUpdates.add(node);
+ ctx.LSNUpdates.add(node);
+ } else if (isLeaf) {
+ // In case of a crash, we un-latch the interior node
+ // inside updateParentForInsert.
+ node.releaseWriteLatch();
+ bufferCache.unpin(node);
+ rightNode.releaseWriteLatch();
+ bufferCache.unpin(rightNode);
+ } else {
+ rightNode.releaseWriteLatch();
+ bufferCache.unpin(rightNode);
+ }
+
+ }
+ ctx.splitKey.setPages(pageId, rightPageId);
+ if (pageId == rootPage) {
+ int newLeftId = freePageManager.getFreePage(ctx.metaFrame);
+ ICachedPage newLeftNode = bufferCache
+ .pin(BufferedFileHandle.getDiskPageId(fileId, newLeftId), true);
+ newLeftNode.acquireWriteLatch();
+ succeeded = false;
+ try {
+ // copy left child to new left child
+ System.arraycopy(node.getBuffer().array(), 0, newLeftNode.getBuffer().array(), 0, newLeftNode
+ .getBuffer().capacity());
+
+ // initialize new root (leftNode becomes new root)
+ ctx.interiorFrame.setPage(node);
+ ctx.interiorFrame.initBuffer((byte) (ctx.interiorFrame.getLevel() + 1));
+
+ ctx.splitKey.setLeftPage(newLeftId);
+ ctx.interiorFrame.insert(ctx.splitKey.getLeftTuple(), -1);
+ ctx.interiorFrame.insert(ctx.splitKey.getRightTuple(), -1);
+
+ succeeded = true;
+ } finally {
+ if (succeeded) {
+ ctx.NSNUpdates.remove(ctx.NSNUpdates.size() - 1);
+ ctx.LSNUpdates.remove(ctx.LSNUpdates.size() - 1);
+
+ ctx.NSNUpdates.add(newLeftNode);
+ ctx.LSNUpdates.add(newLeftNode);
+
+ ctx.NSNUpdates.add(node);
+ ctx.LSNUpdates.add(node);
+ ctx.splitKey.reset();
+ } else if (isLeaf) {
+ // In case of a crash, we un-latch the interior node
+ // inside updateParentForInsert.
+ node.releaseWriteLatch();
+ bufferCache.unpin(node);
+ rightNode.releaseWriteLatch();
+ bufferCache.unpin(rightNode);
+ newLeftNode.releaseWriteLatch();
+ bufferCache.unpin(newLeftNode);
+ } else {
+ rightNode.releaseWriteLatch();
+ bufferCache.unpin(rightNode);
+ newLeftNode.releaseWriteLatch();
+ bufferCache.unpin(newLeftNode);
+ }
+ }
}
break;
}
}
}
- public void updateParentForInsert(RTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ private void updateParentForInsert(RTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
+ boolean succeeded = false;
boolean writeLatched = false;
int parentId = ctx.pathList.getLastPageId();
ICachedPage parentNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, parentId), false);
- incrementPins();
parentNode.acquireWriteLatch();
writeLatched = true;
- incrementWriteLatchesAcquired();
ctx.interiorFrame.setPage(parentNode);
boolean foundParent = true;
try {
-
if (ctx.interiorFrame.getPageLsn() != ctx.pathList.getLastPageLsn()) {
foundParent = false;
while (true) {
- if (ctx.interiorFrame.findTupleByPointer(ctx.splitKey.getLeftTuple(), cmp) != -1) {
+ if (ctx.interiorFrame.findTupleByPointer(ctx.splitKey.getLeftTuple(), ctx.cmp) != -1) {
// found the parent
foundParent = true;
break;
@@ -576,9 +563,7 @@
int rightPage = ctx.interiorFrame.getRightPage();
parentNode.releaseWriteLatch();
writeLatched = false;
- incrementWriteLatchesReleased();
bufferCache.unpin(parentNode);
- incrementUnpins();
if (rightPage == -1) {
break;
@@ -586,46 +571,45 @@
parentId = rightPage;
parentNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, parentId), false);
- incrementPins();
parentNode.acquireWriteLatch();
writeLatched = true;
- incrementWriteLatchesAcquired();
ctx.interiorFrame.setPage(parentNode);
}
}
+
if (foundParent) {
- ctx.interiorFrame.adjustKey(ctx.splitKey.getLeftTuple(), -1, cmp);
+ try {
+ ctx.interiorFrame.adjustKey(ctx.splitKey.getLeftTuple(), -1, ctx.cmp);
+ } catch (TreeIndexException e) {
+ if (writeLatched) {
+ parentNode.releaseWriteLatch();
+ writeLatched = false;
+ bufferCache.unpin(parentNode);
+ }
+ throw e;
+ }
insertTuple(parentNode, parentId, ctx.splitKey.getRightTuple(), ctx, ctx.interiorFrame.isLeaf());
ctx.pathList.moveLast();
-
- parentNode.releaseWriteLatch();
- writeLatched = false;
- incrementWriteLatchesReleased();
- bufferCache.unpin(parentNode);
- incrementUnpins();
+ succeeded = true;
return;
- }
+ }
} finally {
- if (writeLatched) {
- parentNode.releaseWriteLatch();
- writeLatched = false;
- incrementWriteLatchesReleased();
- bufferCache.unpin(parentNode);
- incrementUnpins();
+ if (!succeeded) {
+ if (writeLatched) {
+ parentNode.releaseWriteLatch();
+ writeLatched = false;
+ bufferCache.unpin(parentNode);
+ }
}
}
- // very rare situation when the there is a root split, do an
- // exhaustive
- // breadth-first traversal looking for the parent tuple
- ctx.pathList.clear();
ctx.traverseList.clear();
findPath(ctx);
updateParentForInsert(ctx);
}
- public void findPath(RTreeOpContext ctx) throws HyracksDataException {
+ private void findPath(RTreeOpContext ctx) throws TreeIndexException, HyracksDataException {
boolean readLatched = false;
int pageId = rootPage;
int parentIndex = -1;
@@ -640,10 +624,8 @@
parentIndex = ctx.traverseList.getFirstPageIndex();
node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- incrementPins();
node.acquireReadLatch();
readLatched = true;
- incrementReadLatchesAcquired();
ctx.interiorFrame.setPage(node);
pageLsn = ctx.interiorFrame.getPageLsn();
pageIndex = ctx.traverseList.first();
@@ -651,310 +633,199 @@
ctx.traverseList.moveFirst();
+ if (ctx.interiorFrame.isLeaf()) {
+ throw new TreeIndexException("Error: Failed to re-find parent of a page in the tree.");
+ }
+
+ if (pageId != rootPage) {
+ parentLsn = ctx.traverseList.getPageLsn(ctx.traverseList.getPageIndex(pageIndex));
+ }
if (pageId != rootPage && parentLsn < ctx.interiorFrame.getPageNsn()) {
int rightPage = ctx.interiorFrame.getRightPage();
if (rightPage != -1) {
- ctx.traverseList.add(rightPage, -1, parentIndex);
+ ctx.traverseList.addFirst(rightPage, -1, parentIndex);
}
}
- parentLsn = pageLsn;
- if (ctx.interiorFrame.findTupleByPointer(ctx.splitKey.getLeftTuple(), ctx.traverseList, pageIndex, cmp) != -1) {
+ if (ctx.interiorFrame.findTupleByPointer(ctx.splitKey.getLeftTuple(), ctx.traverseList, pageIndex,
+ ctx.cmp) != -1) {
+ ctx.pathList.clear();
fillPath(ctx, pageIndex);
return;
}
node.releaseReadLatch();
readLatched = false;
- incrementReadLatchesReleased();
bufferCache.unpin(node);
- incrementUnpins();
}
} finally {
if (readLatched) {
node.releaseReadLatch();
readLatched = false;
- incrementReadLatchesReleased();
bufferCache.unpin(node);
- incrementUnpins();
}
}
}
- public void fillPath(RTreeOpContext ctx, int pageIndex) {
+ private void fillPath(RTreeOpContext ctx, int pageIndex) {
if (pageIndex != -1) {
fillPath(ctx, ctx.traverseList.getPageIndex(pageIndex));
ctx.pathList.add(ctx.traverseList.getPageId(pageIndex), ctx.traverseList.getPageLsn(pageIndex), -1);
}
}
- public void delete(ITupleReference tuple, RTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
+ private void delete(ITupleReference tuple, RTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
ctx.reset();
ctx.setTuple(tuple);
ctx.splitKey.reset();
- ctx.splitKey.getLeftTuple().setFieldCount(cmp.getKeyFieldCount());
+ ctx.splitKey.getLeftTuple().setFieldCount(cmpFactories.length);
+ // We delete the first matching tuple (including the payload data.
+ // We don't update the MBRs of the parents after deleting the record.
int tupleIndex = findTupleToDelete(ctx);
if (tupleIndex != -1) {
- int pageId = ctx.pathList.getLastPageId();
- ctx.pathList.moveLast();
- deleteTuple(pageId, tupleIndex, ctx);
-
- while (true) {
- if (ctx.splitKey.getLeftPageBuffer() != null) {
- updateParentForDelete(ctx);
- } else {
- break;
- }
+ try {
+ deleteTuple(tupleIndex, ctx);
+ } finally {
+ ctx.leafFrame.getPage().releaseWriteLatch();
+ bufferCache.unpin(ctx.leafFrame.getPage());
}
-
- ctx.leafFrame.getPage().releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(ctx.leafFrame.getPage());
- incrementUnpins();
}
}
- public void updateParentForDelete(RTreeOpContext ctx) throws HyracksDataException {
- boolean writeLatched = false;
- int parentId = ctx.pathList.getLastPageId();
- ICachedPage parentNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, parentId), false);
- incrementPins();
- parentNode.acquireWriteLatch();
- writeLatched = true;
- incrementWriteLatchesAcquired();
- ctx.interiorFrame.setPage(parentNode);
- boolean foundParent = true;
- int tupleIndex = -1;
-
- try {
- if (ctx.interiorFrame.getPageLsn() != ctx.pathList.getLastPageLsn()) {
- foundParent = false;
- while (true) {
- tupleIndex = ctx.interiorFrame.findTupleByPointer(ctx.splitKey.getLeftTuple(), cmp);
- if (tupleIndex != -1) {
- // found the parent
- foundParent = true;
- break;
- }
- int rightPage = ctx.interiorFrame.getRightPage();
- parentNode.releaseWriteLatch();
- writeLatched = false;
- incrementWriteLatchesReleased();
- bufferCache.unpin(parentNode);
- incrementUnpins();
-
- if (rightPage == -1) {
- break;
- }
-
- parentId = rightPage;
- parentNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, parentId), false);
- incrementPins();
- parentNode.acquireWriteLatch();
- writeLatched = true;
- incrementWriteLatchesAcquired();
- ctx.interiorFrame.setPage(parentNode);
- }
- }
- if (foundParent) {
- if (tupleIndex == -1) {
- tupleIndex = ctx.interiorFrame.findTupleByPointer(ctx.splitKey.getLeftTuple(), cmp);
- }
- boolean recomputeMBR = ctx.interiorFrame.recomputeMBR(ctx.splitKey.getLeftTuple(), tupleIndex, cmp);
-
- if (recomputeMBR) {
- ctx.interiorFrame.adjustKey(ctx.splitKey.getLeftTuple(), tupleIndex, cmp);
- ctx.pathList.moveLast();
-
- incrementGlobalNsn();
- ctx.interiorFrame.setPageLsn(getGlobalNsn());
-
- ctx.splitKey.reset();
- if (!ctx.pathList.isEmpty()) {
- ctx.interiorFrame.computeMBR(ctx.splitKey);
- ctx.splitKey.setLeftPage(parentId);
- }
- } else {
- ctx.pathList.moveLast();
- ctx.splitKey.reset();
- }
-
- parentNode.releaseWriteLatch();
- writeLatched = false;
- incrementWriteLatchesReleased();
- bufferCache.unpin(parentNode);
- incrementUnpins();
- return;
- }
- } finally {
- if (writeLatched) {
- parentNode.releaseWriteLatch();
- writeLatched = false;
- incrementWriteLatchesReleased();
- bufferCache.unpin(parentNode);
- incrementUnpins();
- }
- }
-
- // very rare situation when the there is a root split, do an exhaustive
- // breadth-first traversal looking for the parent tuple
-
- ctx.pathList.clear();
- ctx.traverseList.clear();
- findPath(ctx);
- updateParentForDelete(ctx);
- }
-
- public int findTupleToDelete(RTreeOpContext ctx) throws HyracksDataException {
+ private int findTupleToDelete(RTreeOpContext ctx) throws HyracksDataException {
boolean writeLatched = false;
boolean readLatched = false;
- boolean succeed = false;
+ boolean succeeded = false;
ICachedPage node = null;
- ctx.traverseList.add(rootPage, -1, -1);
- ctx.pathList.add(rootPage, -1, ctx.traverseList.size() - 1);
+ ctx.pathList.add(rootPage, -1, -1);
try {
while (!ctx.pathList.isEmpty()) {
int pageId = ctx.pathList.getLastPageId();
long parentLsn = ctx.pathList.getLastPageLsn();
- int pageIndex = ctx.pathList.getLastPageIndex();
ctx.pathList.moveLast();
node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- incrementPins();
node.acquireReadLatch();
readLatched = true;
- incrementReadLatchesAcquired();
ctx.interiorFrame.setPage(node);
boolean isLeaf = ctx.interiorFrame.isLeaf();
long pageLsn = ctx.interiorFrame.getPageLsn();
- int parentIndex = ctx.traverseList.getPageIndex(pageIndex);
- ctx.traverseList.setPageLsn(pageIndex, pageLsn);
if (pageId != rootPage && parentLsn < ctx.interiorFrame.getPageNsn()) {
// Concurrent split detected, we need to visit the right
// page
int rightPage = ctx.interiorFrame.getRightPage();
if (rightPage != -1) {
- ctx.traverseList.add(rightPage, -1, parentIndex);
- ctx.pathList.add(rightPage, parentLsn, ctx.traverseList.size() - 1);
+ ctx.pathList.add(rightPage, parentLsn, -1);
}
}
if (!isLeaf) {
for (int i = 0; i < ctx.interiorFrame.getTupleCount(); i++) {
- int childPageId = ctx.interiorFrame.getChildPageIdIfIntersect(ctx.tuple, i, cmp);
+ int childPageId = ctx.interiorFrame.getChildPageIdIfIntersect(ctx.tuple, i, ctx.cmp);
if (childPageId != -1) {
- ctx.traverseList.add(childPageId, -1, pageIndex);
- ctx.pathList.add(childPageId, pageLsn, ctx.traverseList.size() - 1);
+ ctx.pathList.add(childPageId, pageLsn, -1);
}
}
} else {
ctx.leafFrame.setPage(node);
- int tupleIndex = ctx.leafFrame.findTupleIndex(ctx.tuple, cmp);
+ int tupleIndex = ctx.leafFrame.findTupleIndex(ctx.tuple, ctx.cmp);
if (tupleIndex != -1) {
node.releaseReadLatch();
readLatched = false;
- incrementReadLatchesReleased();
bufferCache.unpin(node);
- incrementUnpins();
node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- incrementPins();
node.acquireWriteLatch();
writeLatched = true;
- incrementWriteLatchesAcquired();
ctx.leafFrame.setPage(node);
+ // A rare case only happen when a root is no longer a
+ // leaf page. Simply we restart the search.
+ if (!ctx.leafFrame.isLeaf()) {
+ ctx.pathList.add(pageId, -1, -1);
+
+ node.releaseWriteLatch();
+ writeLatched = false;
+ bufferCache.unpin(node);
+ continue;
+ }
+
if (ctx.leafFrame.getPageLsn() != pageLsn) {
// The page was changed while we unlocked it
- tupleIndex = ctx.leafFrame.findTupleIndex(ctx.tuple, cmp);
+ tupleIndex = ctx.leafFrame.findTupleIndex(ctx.tuple, ctx.cmp);
if (tupleIndex == -1) {
- ctx.traverseList.add(pageId, -1, parentIndex);
- ctx.pathList.add(pageId, parentLsn, ctx.traverseList.size() - 1);
+ ctx.pathList.add(pageId, parentLsn, -1);
node.releaseWriteLatch();
writeLatched = false;
- incrementWriteLatchesReleased();
bufferCache.unpin(node);
- incrementUnpins();
continue;
} else {
- ctx.pathList.clear();
- fillPath(ctx, pageIndex);
- succeed = true;
+ succeeded = true;
return tupleIndex;
}
} else {
- ctx.pathList.clear();
- fillPath(ctx, pageIndex);
- succeed = true;
+ succeeded = true;
return tupleIndex;
}
}
}
node.releaseReadLatch();
readLatched = false;
- incrementReadLatchesReleased();
bufferCache.unpin(node);
- incrementUnpins();
}
} finally {
- if (!succeed) {
+ if (!succeeded) {
if (readLatched) {
node.releaseReadLatch();
readLatched = false;
- incrementReadLatchesReleased();
bufferCache.unpin(node);
- incrementUnpins();
} else if (writeLatched) {
node.releaseWriteLatch();
writeLatched = false;
- incrementWriteLatchesReleased();
bufferCache.unpin(node);
- incrementUnpins();
}
}
}
return -1;
}
- public void deleteTuple(int pageId, int tupleIndex, RTreeOpContext ctx) throws HyracksDataException {
- ctx.leafFrame.delete(tupleIndex, cmp);
- incrementGlobalNsn();
- ctx.leafFrame.setPageLsn(getGlobalNsn());
-
- // if the page is empty, just leave it there for future inserts
- if (pageId != rootPage && ctx.leafFrame.getTupleCount() > 0) {
- ctx.leafFrame.computeMBR(ctx.splitKey);
- ctx.splitKey.setLeftPage(pageId);
- }
+ private void deleteTuple(int tupleIndex, RTreeOpContext ctx) throws HyracksDataException {
+ ctx.leafFrame.delete(tupleIndex, ctx.cmp);
+ ctx.leafFrame.setPageLsn(incrementGlobalNsn());
}
- private void search(ITreeIndexCursor cursor, ISearchPredicate searchPred, RTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
- ctx.reset();
+ private void search(ITreeIndexCursor cursor, ISearchPredicate searchPred, RTreeOpContext ctx)
+ throws HyracksDataException, TreeIndexException {
+ ctx.reset();
ctx.cursor = cursor;
cursor.setBufferCache(bufferCache);
cursor.setFileId(fileId);
ctx.cursorInitialState.setRootPage(rootPage);
- ctx.cursor.open(ctx.cursorInitialState, (SearchPredicate)searchPred);
+ ctx.cursor.open(ctx.cursorInitialState, (SearchPredicate) searchPred);
}
+ @Override
public ITreeIndexFrameFactory getInteriorFrameFactory() {
return interiorFrameFactory;
}
+ @Override
public ITreeIndexFrameFactory getLeafFrameFactory() {
return leafFrameFactory;
}
- public MultiComparator getCmp() {
- return cmp;
+ @Override
+ public IBinaryComparatorFactory[] getComparatorFactories() {
+ return cmpFactories;
}
+ @Override
public IFreePageManager getFreePageManager() {
return freePageManager;
}
@@ -963,20 +834,37 @@
throw new UnsupportedOperationException("RTree Update not implemented.");
}
+ public boolean isEmptyTree(IRTreeLeafFrame leafFrame) throws HyracksDataException {
+ ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
+ rootNode.acquireReadLatch();
+ try {
+ leafFrame.setPage(rootNode);
+ if (leafFrame.getLevel() == 0 && leafFrame.getTupleCount() == 0) {
+ return true;
+ } else {
+ return false;
+ }
+ } finally {
+ rootNode.releaseReadLatch();
+ bufferCache.unpin(rootNode);
+ }
+ }
+
public final class BulkLoadContext implements IIndexBulkLoadContext {
public ITreeIndexAccessor indexAccessor;
public BulkLoadContext(float fillFactor, IRTreeFrame leafFrame, IRTreeFrame interiorFrame,
ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
- indexAccessor = createAccessor();
+ indexAccessor = createAccessor();
}
}
@Override
public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws HyracksDataException {
- if (loaded) {
- throw new HyracksDataException("Trying to bulk-load RTree but RTree has already been loaded.");
+ IRTreeLeafFrame leafFrame = (IRTreeLeafFrame) leafFrameFactory.createFrame();
+ if (!isEmptyTree(leafFrame)) {
+ throw new HyracksDataException("Trying to Bulk-load a non-empty RTree.");
}
BulkLoadContext ctx = new BulkLoadContext(fillFactor, (IRTreeFrame) leafFrameFactory.createFrame(),
@@ -988,21 +876,23 @@
@Override
public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException {
try {
- ((BulkLoadContext) ictx).indexAccessor.insert(tuple);
+ ((BulkLoadContext) ictx).indexAccessor.insert(tuple);
} catch (Exception e) {
- throw new HyracksDataException("BulkLoad Error");
+ throw new HyracksDataException("BulkLoad Error", e);
}
}
@Override
public void endBulkLoad(IIndexBulkLoadContext ictx) throws HyracksDataException {
- loaded = true;
}
private void diskOrderScan(ITreeIndexCursor icursor, RTreeOpContext ctx) throws HyracksDataException {
TreeDiskOrderScanCursor cursor = (TreeDiskOrderScanCursor) icursor;
ctx.reset();
+ MultiComparator cmp = MultiComparator.create(cmpFactories);
+ SearchPredicate searchPred = new SearchPredicate(null, cmp);
+
int currentPageId = rootPage + 1;
int maxPageId = freePageManager.getMaxPage(ctx.metaFrame);
@@ -1014,7 +904,7 @@
cursor.setCurrentPageId(currentPageId);
cursor.setMaxPageId(maxPageId);
ctx.cursorInitialState.setPage(page);
- cursor.open(ctx.cursorInitialState, diskOrderScanPredicate);
+ cursor.open(ctx.cursorInitialState, searchPred);
} catch (Exception e) {
page.releaseReadLatch();
bufferCache.unpin(page);
@@ -1036,23 +926,23 @@
public IndexType getIndexType() {
return IndexType.RTREE;
}
-
+
@Override
- public ITreeIndexAccessor createAccessor() {
- return new RTreeAccessor(this);
- }
-
- private class RTreeAccessor implements ITreeIndexAccessor {
+ public ITreeIndexAccessor createAccessor() {
+ return new RTreeAccessor(this);
+ }
+
+ public class RTreeAccessor implements ITreeIndexAccessor {
private RTree rtree;
private RTreeOpContext ctx;
-
+
public RTreeAccessor(RTree rtree) {
this.rtree = rtree;
this.ctx = rtree.createOpContext();
}
-
+
@Override
- public void insert(ITupleReference tuple) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ public void insert(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
ctx.reset(IndexOp.INSERT);
rtree.insert(tuple, ctx);
}
@@ -1070,10 +960,21 @@
}
@Override
- public void search(ITreeIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
- TreeIndexException {
+ public ITreeIndexCursor createSearchCursor() {
+ return new RTreeSearchCursor((IRTreeInteriorFrame) interiorFrameFactory.createFrame(),
+ (IRTreeLeafFrame) leafFrameFactory.createFrame());
+ }
+
+ @Override
+ public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
+ IndexException {
ctx.reset(IndexOp.SEARCH);
- rtree.search(cursor, searchPred, ctx);
+ rtree.search((ITreeIndexCursor) cursor, searchPred, ctx);
+ }
+
+ @Override
+ public ITreeIndexCursor createDiskOrderScanCursor() {
+ return new TreeDiskOrderScanCursor(leafFrameFactory.createFrame());
}
@Override
@@ -1081,5 +982,15 @@
ctx.reset(IndexOp.DISKORDERSCAN);
rtree.diskOrderScan(cursor, ctx);
}
+
+ public RTreeOpContext getOpContext() {
+ return ctx;
+ }
+
+ @Override
+ public void upsert(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
+ throw new UnsupportedOperationException(
+ "The RTree does not suypport the notion of keys, therefore upsert does not make sense.");
+ }
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java
index c258377..6683444 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java
@@ -15,73 +15,85 @@
package edu.uci.ics.hyracks.storage.am.rtree.impls;
+import java.util.ArrayList;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
-public final class RTreeOpContext implements IIndexOpContext {
- public final IRTreeInteriorFrame interiorFrame;
- public final IRTreeLeafFrame leafFrame;
- public IndexOp op;
- public ITreeIndexCursor cursor;
- public RTreeCursorInitialState cursorInitialState;
- public ITreeIndexMetaDataFrame metaFrame;
- public RTreeSplitKey splitKey;
- public ITupleReference tuple;
- public PathList pathList; // used to record the pageIds and pageLsns
- // of the visited pages
- public PathList traverseList; // used for traversing the tree
- private static final int initTraverseListSize = 100;
+public class RTreeOpContext implements IIndexOpContext {
+ private static final int INITIAL_TRAVERSE_LIST_SIZE = 100;
+ public final MultiComparator cmp;
+ public final IRTreeInteriorFrame interiorFrame;
+ public final IRTreeLeafFrame leafFrame;
+ public IndexOp op;
+ public ITreeIndexCursor cursor;
+ public RTreeCursorInitialState cursorInitialState;
+ public ITreeIndexMetaDataFrame metaFrame;
+ public RTreeSplitKey splitKey;
+ public ITupleReference tuple;
+ // Used to record the pageIds and pageLsns of the visited pages.
+ public PathList pathList;
+ // Used for traversing the tree.
+ public PathList traverseList;
- public RTreeOpContext(IRTreeLeafFrame leafFrame,
- IRTreeInteriorFrame interiorFrame,
- ITreeIndexMetaDataFrame metaFrame, int treeHeightHint) {
- this.interiorFrame = interiorFrame;
- this.leafFrame = leafFrame;
- this.metaFrame = metaFrame;
- pathList = new PathList(treeHeightHint, treeHeightHint);
- }
+ public ArrayList<ICachedPage> NSNUpdates;
+ public ArrayList<ICachedPage> LSNUpdates;
- public ITupleReference getTuple() {
- return tuple;
- }
+ public RTreeOpContext(IRTreeLeafFrame leafFrame, IRTreeInteriorFrame interiorFrame,
+ ITreeIndexMetaDataFrame metaFrame, IBinaryComparatorFactory[] cmpFactories, int treeHeightHint) {
+ this.cmp = MultiComparator.create(cmpFactories);
+ this.interiorFrame = interiorFrame;
+ this.leafFrame = leafFrame;
+ this.metaFrame = metaFrame;
+ pathList = new PathList(treeHeightHint, treeHeightHint);
+ NSNUpdates = new ArrayList<ICachedPage>();
+ LSNUpdates = new ArrayList<ICachedPage>();
+ }
- public void setTuple(ITupleReference tuple) {
- this.tuple = tuple;
- }
+ public ITupleReference getTuple() {
+ return tuple;
+ }
- public void reset() {
- if (pathList != null) {
- pathList.clear();
- }
- if (traverseList != null) {
- traverseList.clear();
- }
- }
+ public void setTuple(ITupleReference tuple) {
+ this.tuple = tuple;
+ }
- @Override
- public void reset(IndexOp newOp) {
- if (op != null && newOp == op) {
- return;
- }
- if (op != IndexOp.SEARCH && op != IndexOp.DISKORDERSCAN) {
- if (splitKey == null) {
- splitKey = new RTreeSplitKey(interiorFrame.getTupleWriter()
- .createTupleReference(), interiorFrame.getTupleWriter()
- .createTupleReference());
- }
- if (traverseList == null) {
- traverseList = new PathList(initTraverseListSize,
- initTraverseListSize);
- }
- }
- if (cursorInitialState == null) {
- cursorInitialState = new RTreeCursorInitialState(pathList, 1);
- }
- this.op = newOp;
- }
+ public void reset() {
+ if (pathList != null) {
+ pathList.clear();
+ }
+ if (traverseList != null) {
+ traverseList.clear();
+ }
+ NSNUpdates.clear();
+ LSNUpdates.clear();
+ }
+
+ @Override
+ public void reset(IndexOp newOp) {
+ if (op != null && newOp == op) {
+ return;
+ }
+ if (op != IndexOp.SEARCH && op != IndexOp.DISKORDERSCAN) {
+ if (splitKey == null) {
+ splitKey = new RTreeSplitKey(interiorFrame.getTupleWriter().createTupleReference(), interiorFrame
+ .getTupleWriter().createTupleReference());
+ }
+ if (traverseList == null) {
+ traverseList = new PathList(INITIAL_TRAVERSE_LIST_SIZE, INITIAL_TRAVERSE_LIST_SIZE);
+ }
+ }
+ if (cursorInitialState == null) {
+ cursorInitialState = new RTreeCursorInitialState(pathList, 1);
+ }
+ this.op = newOp;
+ }
}
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
index e65f1d4..ee7ec5f 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
@@ -39,19 +39,18 @@
private SearchPredicate pred;
private PathList pathList;
private int rootPage;
- ITupleReference searchKey;
+ private ITupleReference searchKey;
private int tupleIndex = 0;
private int tupleIndexInc = 0;
+ private int currentTupleIndex = 0;
+ private int pageId = -1;
private MultiComparator cmp;
private ITreeIndexTupleReference frameTuple;
private boolean readLatched = false;
- private int pin = 0;
- private int unpin = 0;
-
public RTreeSearchCursor(IRTreeInteriorFrame interiorFrame, IRTreeLeafFrame leafFrame) {
this.interiorFrame = interiorFrame;
this.leafFrame = leafFrame;
@@ -59,7 +58,7 @@
}
@Override
- public void close() throws Exception {
+ public void close() throws HyracksDataException {
if (readLatched) {
page.releaseReadLatch();
bufferCache.unpin(page);
@@ -75,17 +74,24 @@
return frameTuple;
}
+ public int getTupleOffset() {
+ return leafFrame.getTupleOffset(currentTupleIndex);
+ }
+
+ public int getPageId() {
+ return pageId;
+ }
+
@Override
public ICachedPage getPage() {
return page;
}
- public boolean fetchNextLeafPage() throws HyracksDataException {
- boolean succeed = false;
+ private boolean fetchNextLeafPage() throws HyracksDataException {
+ boolean succeeded = false;
if (readLatched) {
page.releaseReadLatch();
bufferCache.unpin(page);
- unpin++;
readLatched = false;
}
@@ -94,7 +100,6 @@
long parentLsn = pathList.getLastPageLsn();
pathList.moveLast();
ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- pin++;
node.acquireReadLatch();
readLatched = true;
try {
@@ -112,26 +117,35 @@
}
if (!isLeaf) {
- for (int i = 0; i < interiorFrame.getTupleCount(); i++) {
- int childPageId = interiorFrame.getChildPageIdIfIntersect(searchKey, i, cmp);
- if (childPageId != -1) {
+ if (searchKey != null) {
+ for (int i = 0; i < interiorFrame.getTupleCount(); i++) {
+ int childPageId = interiorFrame.getChildPageIdIfIntersect(searchKey, i, cmp);
+ if (childPageId != -1) {
+ pathList.add(childPageId, pageLsn, -1);
+ }
+ }
+ } else {
+ for (int i = 0; i < interiorFrame.getTupleCount(); i++) {
+ int childPageId = interiorFrame.getChildPageId(i);
pathList.add(childPageId, pageLsn, -1);
}
}
+
} else {
page = node;
+ this.pageId = pageId; // This is only needed for the
+ // LSMRTree flush operation
leafFrame.setPage(page);
tupleIndex = 0;
- succeed = true;
+ succeeded = true;
return true;
}
} finally {
- if (!succeed) {
+ if (!succeeded) {
if (readLatched) {
node.releaseReadLatch();
readLatched = false;
bufferCache.unpin(node);
- unpin++;
}
}
}
@@ -140,7 +154,7 @@
}
@Override
- public boolean hasNext() throws Exception {
+ public boolean hasNext() throws HyracksDataException {
if (page == null) {
return false;
}
@@ -153,8 +167,19 @@
do {
for (int i = tupleIndex; i < leafFrame.getTupleCount(); i++) {
- if (leafFrame.intersect(searchKey, i, cmp)) {
+ if (searchKey != null) {
+ if (leafFrame.intersect(searchKey, i, cmp)) {
+ frameTuple.resetByTupleIndex(leafFrame, i);
+ currentTupleIndex = i; // This is only needed for the
+ // LSMRTree flush operation
+ tupleIndexInc = i + 1;
+ return true;
+ }
+ } else {
frameTuple.resetByTupleIndex(leafFrame, i);
+ currentTupleIndex = i; // This is only needed for the
+ // LSMRTree
+ // flush operation
tupleIndexInc = i + 1;
return true;
}
@@ -164,7 +189,7 @@
}
@Override
- public void next() throws Exception {
+ public void next() throws HyracksDataException {
tupleIndex = tupleIndexInc;
}
@@ -185,14 +210,17 @@
cmp = pred.getLowKeyComparator();
searchKey = pred.getSearchKey();
- int maxFieldPos = cmp.getKeyFieldCount() / 2;
- for (int i = 0; i < maxFieldPos; i++) {
- int j = maxFieldPos + i;
- int c = cmp.getComparators()[i].compare(searchKey.getFieldData(i), searchKey.getFieldStart(i),
- searchKey.getFieldLength(i), searchKey.getFieldData(j), searchKey.getFieldStart(j),
- searchKey.getFieldLength(j));
- if (c > 0) {
- throw new IllegalArgumentException("The low key point has larger coordinates than the high key point.");
+ if (searchKey != null) {
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ int c = cmp.getComparators()[i].compare(searchKey.getFieldData(i), searchKey.getFieldStart(i),
+ searchKey.getFieldLength(i), searchKey.getFieldData(j), searchKey.getFieldStart(j),
+ searchKey.getFieldLength(j));
+ if (c > 0) {
+ throw new IllegalArgumentException(
+ "The low key point has larger coordinates than the high key point.");
+ }
}
}
@@ -220,8 +248,8 @@
this.fileId = fileId;
}
- @Override
- public boolean exclusiveLatchNodes() {
- return false;
- }
+ @Override
+ public boolean exclusiveLatchNodes() {
+ return false;
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/UnorderedSlotManager.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/UnorderedSlotManager.java
index 260cdea..a2dcb6d 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/UnorderedSlotManager.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/UnorderedSlotManager.java
@@ -24,105 +24,100 @@
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMFrame;
public class UnorderedSlotManager extends AbstractSlotManager {
-
- @Override
- public int findTupleIndex(ITupleReference searchKey,
- ITreeIndexTupleReference frameTuple, MultiComparator multiCmp,
- FindTupleMode mode, FindTupleNoExactMatchPolicy matchPolicy) {
- int maxFieldPos = multiCmp.getKeyFieldCount() / 2;
- for (int i = 0; i < frame.getTupleCount(); i++) {
- frameTuple.resetByTupleIndex(frame, i);
+ @Override
+ public int findTupleIndex(ITupleReference searchKey, ITreeIndexTupleReference frameTuple, MultiComparator multiCmp,
+ FindTupleMode mode, FindTupleNoExactMatchPolicy matchPolicy) {
+ if (searchKey.getFieldCount() == frameTuple.getFieldCount()) {
+ int maxFieldPos = multiCmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < frame.getTupleCount(); i++) {
+ frameTuple.resetByTupleIndex(frame, i);
- boolean foundTuple = true;
- for (int j = 0; j < maxFieldPos; j++) {
- int k = maxFieldPos + j;
- int c1 = multiCmp.getComparators()[j].compare(
- frameTuple.getFieldData(j),
- frameTuple.getFieldStart(j),
- frameTuple.getFieldLength(j),
- searchKey.getFieldData(j), searchKey.getFieldStart(j),
- searchKey.getFieldLength(j));
+ boolean foundTuple = true;
+ for (int j = 0; j < maxFieldPos; j++) {
+ int k = maxFieldPos + j;
+ int c1 = multiCmp.getComparators()[j].compare(frameTuple.getFieldData(j),
+ frameTuple.getFieldStart(j), frameTuple.getFieldLength(j), searchKey.getFieldData(j),
+ searchKey.getFieldStart(j), searchKey.getFieldLength(j));
- if (c1 != 0) {
- foundTuple = false;
- break;
- }
- int c2 = multiCmp.getComparators()[k].compare(
- frameTuple.getFieldData(k),
- frameTuple.getFieldStart(k),
- frameTuple.getFieldLength(k),
- searchKey.getFieldData(k), searchKey.getFieldStart(k),
- searchKey.getFieldLength(k));
- if (c2 != 0) {
- foundTuple = false;
- break;
- }
- }
- int remainingFieldCount = frameTuple.getFieldCount()
- - multiCmp.getKeyFieldCount();
- for (int j = multiCmp.getKeyFieldCount(); j < multiCmp
- .getKeyFieldCount() + remainingFieldCount; j++) {
- if (!compareField(searchKey, frameTuple, j)) {
- foundTuple = false;
- break;
- }
- }
- if (foundTuple) {
- return i;
- }
- }
- return -1;
- }
+ if (c1 != 0) {
+ foundTuple = false;
+ break;
+ }
+ int c2 = multiCmp.getComparators()[k].compare(frameTuple.getFieldData(k),
+ frameTuple.getFieldStart(k), frameTuple.getFieldLength(k), searchKey.getFieldData(k),
+ searchKey.getFieldStart(k), searchKey.getFieldLength(k));
+ if (c2 != 0) {
+ foundTuple = false;
+ break;
+ }
+ }
+ int remainingFieldCount = frameTuple.getFieldCount() - multiCmp.getKeyFieldCount();
+ for (int j = multiCmp.getKeyFieldCount(); j < multiCmp.getKeyFieldCount() + remainingFieldCount; j++) {
+ if (!compareField(searchKey, frameTuple, j)) {
+ foundTuple = false;
+ break;
+ }
+ }
+ if (foundTuple) {
+ return i;
+ }
+ }
+ }
+ return -1;
+ }
- public boolean compareField(ITupleReference searchKey,
- ITreeIndexTupleReference frameTuple, int fIdx) {
- int searchKeyFieldLength = searchKey.getFieldLength(fIdx);
- int frameTupleFieldLength = frameTuple.getFieldLength(fIdx);
+ private boolean compareField(ITupleReference searchKey, ITreeIndexTupleReference frameTuple, int fIdx) {
+ int searchKeyFieldLength = searchKey.getFieldLength(fIdx);
+ int frameTupleFieldLength = frameTuple.getFieldLength(fIdx);
- if (searchKeyFieldLength != frameTupleFieldLength) {
- return false;
- }
+ if (searchKeyFieldLength != frameTupleFieldLength) {
+ return false;
+ }
- for (int i = 0; i < searchKeyFieldLength; i++) {
- if (searchKey.getFieldData(fIdx)[i + searchKey.getFieldStart(fIdx)] != frameTuple
- .getFieldData(fIdx)[i + frameTuple.getFieldStart(fIdx)]) {
- return false;
- }
- }
- return true;
- }
+ for (int i = 0; i < searchKeyFieldLength; i++) {
+ if (searchKey.getFieldData(fIdx)[i + searchKey.getFieldStart(fIdx)] != frameTuple.getFieldData(fIdx)[i
+ + frameTuple.getFieldStart(fIdx)]) {
+ return false;
+ }
+ }
+ return true;
+ }
- @Override
- public int insertSlot(int tupleIndex, int tupleOff) {
- int slotOff = getSlotEndOff() - slotSize;
- setSlot(slotOff, tupleOff);
- return slotOff;
- }
+ @Override
+ public int insertSlot(int tupleIndex, int tupleOff) {
+ int slotOff = getSlotEndOff() - slotSize;
+ setSlot(slotOff, tupleOff);
+ return slotOff;
+ }
- public void modifySlot(int slotOff, int tupleOff) {
- setSlot(slotOff, tupleOff);
- }
+ public void modifySlot(int slotOff, int tupleOff) {
+ setSlot(slotOff, tupleOff);
+ }
- public void deleteEmptySlots() {
- int slotOff = getSlotStartOff();
- while (slotOff >= getSlotEndOff()) {
- if (frame.getBuffer().getInt(slotOff) == -1) {
- while (frame.getBuffer().getInt(getSlotEndOff()) == -1) {
- ((RTreeNSMFrame) frame)
- .setTupleCount(frame.getTupleCount() - 1);
- }
- if (slotOff > getSlotEndOff()) {
- System.arraycopy(frame.getBuffer().array(),
- getSlotEndOff(), frame.getBuffer().array(),
- slotOff, slotSize);
- ((RTreeNSMFrame) frame)
- .setTupleCount(frame.getTupleCount() - 1);
- } else {
- break;
- }
- }
- slotOff -= slotSize;
- }
- }
+ public void deleteSlot(int slotOff) {
+ System.arraycopy(frame.getBuffer().array(), getSlotEndOff(), frame.getBuffer().array(), slotOff + slotSize,
+ slotSize);
+ }
+
+ public void deleteEmptySlots() {
+ int slotOff = getSlotStartOff();
+ while (frame.getTupleCount() > 0) {
+ while (frame.getBuffer().getInt(getSlotEndOff()) == -1) {
+ ((RTreeNSMFrame) frame).setTupleCount(frame.getTupleCount() - 1);
+ if (frame.getTupleCount() == 0) {
+ break;
+ }
+ }
+ if (frame.getTupleCount() == 0 || slotOff <= getSlotEndOff()) {
+ break;
+ }
+ if (frame.getBuffer().getInt(slotOff) == -1) {
+ modifySlot(slotOff, frame.getBuffer().getInt(getSlotEndOff()));
+ ((RTreeNSMFrame) frame).setTupleCount(frame.getTupleCount() - 1);
+ }
+ slotOff -= slotSize;
+
+ }
+ }
}
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/util/RTreeUtils.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/util/RTreeUtils.java
index 74e4840..6f5d36f 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/util/RTreeUtils.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/util/RTreeUtils.java
@@ -15,11 +15,55 @@
package edu.uci.ics.hyracks.storage.am.rtree.util;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.data.std.api.IPointableFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.data.PointablePrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
+import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
public class RTreeUtils {
+ public static RTree createRTree(IBufferCache bufferCache, ITypeTraits[] typeTraits,
+ IPrimitiveValueProviderFactory[] valueProviderFactories, IBinaryComparatorFactory[] cmpFactories) {
+
+ RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
+ ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
+ RTree rtree = new RTree(bufferCache, typeTraits.length, cmpFactories, freePageManager, interiorFrameFactory,
+ leafFrameFactory);
+ return rtree;
+ }
+
+ // Creates a new MultiComparator by constructing new IBinaryComparators.
+ public static MultiComparator getSearchMultiComparator(IBinaryComparatorFactory[] cmpFactories,
+ ITupleReference searchKey) {
+ if (searchKey == null || cmpFactories.length == searchKey.getFieldCount()) {
+ return MultiComparator.create(cmpFactories);
+ }
+ IBinaryComparator[] newCmps = new IBinaryComparator[searchKey.getFieldCount()];
+ for (int i = 0; i < searchKey.getFieldCount(); i++) {
+ newCmps[i] = cmpFactories[i].createBinaryComparator();
+ }
+ return new MultiComparator(newCmps);
+ }
+
public static IPrimitiveValueProviderFactory[] createPrimitiveValueProviderFactories(int len, IPointableFactory pf) {
IPrimitiveValueProviderFactory[] pvpfs = new IPrimitiveValueProviderFactory[len];
for (int i = 0; i < len; ++i) {
@@ -27,4 +71,4 @@
}
return pvpfs;
}
-}
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java b/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java
index a6f1a4b..e170a95 100644
--- a/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java
+++ b/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java
@@ -14,7 +14,6 @@
*/
package edu.uci.ics.hyracks.storage.common.buffercache;
-import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Map;
@@ -28,8 +27,8 @@
import java.util.logging.Logger;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileHandle;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IFileHandle;
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
import edu.uci.ics.hyracks.storage.common.file.IFileMapManager;
@@ -38,12 +37,11 @@
private static final Logger LOGGER = Logger.getLogger(BufferCache.class.getName());
private static final int MAP_FACTOR = 2;
- private static final int MAX_VICTIMIZATION_TRY_COUNT = 5;
- private static final int MAX_WAIT_FOR_CLEANER_THREAD_TIME = 1000;
- private static final int MIN_CLEANED_COUNT_DIFF = 4;
-
- private final int maxOpenFiles;
+ private static final int MIN_CLEANED_COUNT_DIFF = 3;
+ private static final int PIN_MAX_WAIT_TIME = 50;
+ private final int maxOpenFiles;
+
private final IIOManager ioManager;
private final int pageSize;
private final int numPages;
@@ -52,8 +50,8 @@
private final IPageReplacementStrategy pageReplacementStrategy;
private final IFileMapManager fileMapManager;
private final CleanerThread cleanerThread;
- private final Map<Integer, BufferedFileHandle> fileInfoMap;
-
+ private final Map<Integer, BufferedFileHandle> fileInfoMap;
+
private boolean closed;
public BufferCache(IIOManager ioManager, ICacheMemoryAllocator allocator,
@@ -97,10 +95,10 @@
}
// check whether file has been created and opened
- int fileId = BufferedFileHandle.getFileId(dpid);
+ int fileId = BufferedFileHandle.getFileId(dpid);
BufferedFileHandle fInfo = null;
- synchronized(fileInfoMap) {
- fInfo = fileInfoMap.get(fileId);
+ synchronized (fileInfoMap) {
+ fInfo = fileInfoMap.get(fileId);
}
if (fInfo == null) {
throw new HyracksDataException("pin called on a fileId " + fileId + " that has not been created.");
@@ -112,7 +110,6 @@
@Override
public ICachedPage tryPin(long dpid) throws HyracksDataException {
pinSanityCheck(dpid);
-
CachedPage cPage = null;
int hash = hash(dpid);
CacheBucket bucket = pageMap[hash];
@@ -121,7 +118,7 @@
cPage = bucket.cachedPage;
while (cPage != null) {
if (cPage.dpid == dpid) {
- cPage.pinCount.incrementAndGet();
+ cPage.pinCount.incrementAndGet();
pageReplacementStrategy.notifyCachePageAccess(cPage);
return cPage;
}
@@ -130,22 +127,13 @@
} finally {
bucket.bucketLock.unlock();
}
-
return cPage;
}
@Override
public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException {
pinSanityCheck(dpid);
-
CachedPage cPage = findPage(dpid, newPage);
- if (cPage == null) {
- if (LOGGER.isLoggable(Level.FINE)) {
- LOGGER.fine(dumpState());
- }
- throw new HyracksDataException("Failed to pin page " + BufferedFileHandle.getFileId(dpid) + ":"
- + BufferedFileHandle.getPageId(dpid) + " because all pages are pinned.");
- }
if (!newPage) {
if (!cPage.valid) {
/*
@@ -170,10 +158,9 @@
}
private CachedPage findPage(long dpid, boolean newPage) {
- int victimizationTryCount = 0;
while (true) {
- int startCleanedCount = cleanerThread.cleanedCount;
-
+ int startCleanedCount = cleanerThread.cleanedCount;
+
CachedPage cPage = null;
/*
* Hash dpid to get a bucket and then check if the page exists in the bucket.
@@ -185,7 +172,7 @@
cPage = bucket.cachedPage;
while (cPage != null) {
if (cPage.dpid == dpid) {
- cPage.pinCount.incrementAndGet();
+ cPage.pinCount.incrementAndGet();
return cPage;
}
cPage = cPage.next;
@@ -227,8 +214,8 @@
cPage = bucket.cachedPage;
while (cPage != null) {
if (cPage.dpid == dpid) {
- cPage.pinCount.incrementAndGet();
- victim.pinCount.decrementAndGet();
+ cPage.pinCount.incrementAndGet();
+ victim.pinCount.decrementAndGet();
return cPage;
}
cPage = cPage.next;
@@ -249,14 +236,14 @@
bucket.bucketLock.lock();
try {
if (victim.pinCount.get() != 1) {
- victim.pinCount.decrementAndGet();
+ victim.pinCount.decrementAndGet();
continue;
}
cPage = bucket.cachedPage;
while (cPage != null) {
if (cPage.dpid == dpid) {
- cPage.pinCount.incrementAndGet();
- victim.pinCount.decrementAndGet();
+ cPage.pinCount.incrementAndGet();
+ victim.pinCount.decrementAndGet();
return cPage;
}
cPage = cPage.next;
@@ -280,14 +267,14 @@
}
try {
if (victim.pinCount.get() != 1) {
- victim.pinCount.decrementAndGet();
+ victim.pinCount.decrementAndGet();
continue;
}
cPage = bucket.cachedPage;
while (cPage != null) {
if (cPage.dpid == dpid) {
- cPage.pinCount.incrementAndGet();
- victim.pinCount.decrementAndGet();
+ cPage.pinCount.incrementAndGet();
+ victim.pinCount.decrementAndGet();
return cPage;
}
cPage = cPage.next;
@@ -312,28 +299,22 @@
return victim;
}
}
- /*
- * Victimization failed -- all pages pinned? wait a bit, increment victimizationTryCount and loop around. Give up after MAX_VICTIMIZATION_TRY_COUNT trys.
- */
- if (++victimizationTryCount >= MAX_VICTIMIZATION_TRY_COUNT) {
- return null;
- }
synchronized (cleanerThread) {
cleanerThread.notifyAll();
}
- // Heuristic optimization. Check whether the cleaner thread has
- // cleaned pages since we did our last pin attempt.
- if (cleanerThread.cleanedCount - startCleanedCount > MIN_CLEANED_COUNT_DIFF) {
- // Don't go to sleep and wait for notification from the cleaner,
- // just try to pin again immediately.
- continue;
- }
+ // Heuristic optimization. Check whether the cleaner thread has
+ // cleaned pages since we did our last pin attempt.
+ if (cleanerThread.cleanedCount - startCleanedCount > MIN_CLEANED_COUNT_DIFF) {
+ // Don't go to sleep and wait for notification from the cleaner,
+ // just try to pin again immediately.
+ continue;
+ }
synchronized (cleanerThread.cleanNotification) {
- try {
- cleanerThread.cleanNotification.wait(MAX_WAIT_FOR_CLEANER_THREAD_TIME);
- } catch (InterruptedException e) {
- // Do nothing
- }
+ try {
+ cleanerThread.cleanNotification.wait(PIN_MAX_WAIT_TIME);
+ } catch (InterruptedException e) {
+ // Do nothing
+ }
}
}
}
@@ -390,7 +371,7 @@
private void write(CachedPage cPage) throws HyracksDataException {
BufferedFileHandle fInfo = getFileInfo(cPage);
- if(fInfo.fileHasBeenDeleted()){
+ if (fInfo.fileHasBeenDeleted()) {
return;
}
cPage.buffer.position(0);
@@ -512,11 +493,11 @@
private boolean shutdownStart = false;
private boolean shutdownComplete = false;
private final Object cleanNotification = new Object();
- // Simply keeps incrementing this counter when a page is cleaned.
- // Used to implement wait-for-cleanerthread heuristic optimizations.
- // A waiter can detect whether pages have been cleaned.
- // No need to make this var volatile or synchronize it's access in any
- // way because it is used for heuristics.
+ // Simply keeps incrementing this counter when a page is cleaned.
+ // Used to implement wait-for-cleanerthread heuristic optimizations.
+ // A waiter can detect whether pages have been cleaned.
+ // No need to make this var volatile or synchronize it's access in any
+ // way because it is used for heuristics.
private int cleanedCount = 0;
public CleanerThread() {
@@ -524,54 +505,54 @@
}
public void cleanPage(CachedPage cPage, boolean force) {
- if (cPage.dirty.get()) {
- boolean proceed = false;
- if (force) {
- cPage.latch.writeLock().lock();
- proceed = true;
- } else {
- proceed = cPage.latch.readLock().tryLock();
- }
+ if (cPage.dirty.get()) {
+ boolean proceed = false;
+ if (force) {
+ cPage.latch.writeLock().lock();
+ proceed = true;
+ } else {
+ proceed = cPage.latch.readLock().tryLock();
+ }
if (proceed) {
- try {
- // Make sure page is still dirty.
- if (!cPage.dirty.get()) {
- return;
- }
+ try {
+ // Make sure page is still dirty.
+ if (!cPage.dirty.get()) {
+ return;
+ }
boolean cleaned = true;
try {
write(cPage);
} catch (HyracksDataException e) {
cleaned = false;
}
- if (cleaned) {
- cPage.dirty.set(false);
- cPage.pinCount.decrementAndGet();
- cleanedCount++;
- synchronized (cleanNotification) {
- cleanNotification.notifyAll();
- }
+ if (cleaned) {
+ cPage.dirty.set(false);
+ cPage.pinCount.decrementAndGet();
+ cleanedCount++;
+ synchronized (cleanNotification) {
+ cleanNotification.notifyAll();
+ }
}
} finally {
if (force) {
- cPage.latch.writeLock().unlock();
+ cPage.latch.writeLock().unlock();
} else {
- cPage.latch.readLock().unlock();
+ cPage.latch.readLock().unlock();
}
}
} else if (shutdownStart) {
- throw new IllegalStateException(
- "Cache closed, but unable to acquire read lock on dirty page: " + cPage.dpid);
+ throw new IllegalStateException("Cache closed, but unable to acquire read lock on dirty page: "
+ + cPage.dpid);
}
}
}
-
+
@Override
public synchronized void run() {
try {
while (true) {
- for (int i = 0; i < numPages; ++i) {
- CachedPage cPage = cachedPages[i];
+ for (int i = 0; i < numPages; ++i) {
+ CachedPage cPage = cachedPages[i];
cleanPage(cPage, false);
}
if (shutdownStart) {
@@ -592,7 +573,7 @@
@Override
public void close() {
- closed = true;
+ closed = true;
synchronized (cleanerThread) {
cleanerThread.shutdownStart = true;
cleanerThread.notifyAll();
@@ -668,7 +649,7 @@
// create, open, and map new file reference
FileReference fileRef = fileMapManager.lookupFileName(fileId);
- FileHandle fh = ioManager.open(fileRef, IIOManager.FileReadWriteMode.READ_WRITE,
+ IFileHandle fh = ioManager.open(fileRef, IIOManager.FileReadWriteMode.READ_WRITE,
IIOManager.FileSyncMode.METADATA_ASYNC_DATA_ASYNC);
fInfo = new BufferedFileHandle(fileId, fh);
fileInfoMap.put(fileId, fInfo);
@@ -717,7 +698,7 @@
if (flushDirtyPages) {
write(cPage);
}
- cPage.dirty.set(false);
+ cPage.dirty.set(false);
pinCount = cPage.pinCount.decrementAndGet();
} else {
pinCount = cPage.pinCount.get();
@@ -736,10 +717,10 @@
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Closing file: " + fileId + " in cache: " + this);
}
- if (LOGGER.isLoggable(Level.FINE)) {
+ if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine(dumpState());
}
-
+
synchronized (fileInfoMap) {
BufferedFileHandle fInfo = fileInfoMap.get(fileId);
if (fInfo == null) {
@@ -756,32 +737,28 @@
@Override
public void flushDirtyPage(ICachedPage page) throws HyracksDataException {
- // Assumes the caller has pinned the page.
- cleanerThread.cleanPage((CachedPage)page, true);
+ // Assumes the caller has pinned the page.
+ cleanerThread.cleanPage((CachedPage) page, true);
}
-
- @Override
+
+ @Override
public void force(int fileId, boolean metadata) throws HyracksDataException {
- BufferedFileHandle fInfo = null;
- synchronized (fileInfoMap) {
- fInfo = fileInfoMap.get(fileId);
- try {
- fInfo.getFileHandle().getFileChannel().force(metadata);
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- }
+ BufferedFileHandle fInfo = null;
+ synchronized (fileInfoMap) {
+ fInfo = fileInfoMap.get(fileId);
+ ioManager.sync(fInfo.getFileHandle(), metadata);
+ }
}
-
+
@Override
public synchronized void deleteFile(int fileId, boolean flushDirtyPages) throws HyracksDataException {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Deleting file: " + fileId + " in cache: " + this);
}
if (flushDirtyPages) {
- synchronized (fileInfoMap) {
- sweepAndFlush(fileId, flushDirtyPages);
- }
+ synchronized (fileInfoMap) {
+ sweepAndFlush(fileId, flushDirtyPages);
+ }
}
synchronized (fileInfoMap) {
BufferedFileHandle fInfo = null;
diff --git a/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/BufferedFileHandle.java b/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/BufferedFileHandle.java
index ac062d2..1444886b 100644
--- a/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/BufferedFileHandle.java
+++ b/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/BufferedFileHandle.java
@@ -16,14 +16,14 @@
import java.util.concurrent.atomic.AtomicInteger;
-import edu.uci.ics.hyracks.api.io.FileHandle;
+import edu.uci.ics.hyracks.api.io.IFileHandle;
public class BufferedFileHandle {
private final int fileId;
- private FileHandle handle;
+ private IFileHandle handle;
private final AtomicInteger refCount;
- public BufferedFileHandle(int fileId, FileHandle handle) {
+ public BufferedFileHandle(int fileId, IFileHandle handle) {
this.fileId = fileId;
this.handle = handle;
refCount = new AtomicInteger();
@@ -33,18 +33,18 @@
return fileId;
}
- public FileHandle getFileHandle() {
+ public IFileHandle getFileHandle() {
return handle;
}
public void markAsDeleted() {
- handle = null;
+ handle = null;
}
-
+
public boolean fileHasBeenDeleted() {
- return handle == null;
+ return handle == null;
}
-
+
public int incReferenceCount() {
return refCount.incrementAndGet();
}
@@ -52,7 +52,7 @@
public int decReferenceCount() {
return refCount.decrementAndGet();
}
-
+
public int getReferenceCount() {
return refCount.get();
}
diff --git a/hyracks/hyracks-test-support/pom.xml b/hyracks/hyracks-test-support/pom.xml
index b35aae3..12aa45d 100644
--- a/hyracks/hyracks-test-support/pom.xml
+++ b/hyracks/hyracks-test-support/pom.xml
@@ -45,10 +45,24 @@
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-rtree</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-invertedindex</artifactId>
<version>0.2.1-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexBulkLoadTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexBulkLoadTest.java
new file mode 100644
index 0000000..85bfdd2
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexBulkLoadTest.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+
+@SuppressWarnings("rawtypes")
+public abstract class OrderedIndexBulkLoadTest extends OrderedIndexTestDriver {
+
+ private final OrderedIndexTestUtils orderedIndexTestUtils;
+ private final int bulkLoadRounds;
+
+ public OrderedIndexBulkLoadTest(BTreeLeafFrameType[] leafFrameTypesToTest, int bulkLoadRounds) {
+ super(leafFrameTypesToTest);
+ this.bulkLoadRounds = bulkLoadRounds;
+ this.orderedIndexTestUtils = new OrderedIndexTestUtils();
+ }
+
+ @Override
+ protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType,
+ ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey)
+ throws Exception {
+ OrderedIndexTestContext ctx = createTestContext(fieldSerdes, numKeys, leafType);
+ for (int i = 0; i < bulkLoadRounds; i++) {
+ // We assume all fieldSerdes are of the same type. Check the first
+ // one
+ // to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ orderedIndexTestUtils.bulkLoadIntTuples(ctx, numTuplesToInsert, getRandom());
+ } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
+ orderedIndexTestUtils.bulkLoadStringTuples(ctx, numTuplesToInsert, getRandom());
+ }
+ orderedIndexTestUtils.checkPointSearches(ctx);
+ orderedIndexTestUtils.checkScan(ctx);
+ orderedIndexTestUtils.checkDiskOrderScan(ctx);
+ orderedIndexTestUtils.checkRangeSearch(ctx, lowKey, highKey, true, true);
+ if (prefixLowKey != null && prefixHighKey != null) {
+ orderedIndexTestUtils.checkRangeSearch(ctx, prefixLowKey, prefixHighKey, true, true);
+ }
+ }
+ ctx.getIndex().close();
+ }
+
+ @Override
+ protected String getTestOpName() {
+ return "BulkLoad";
+ }
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexDeleteTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexDeleteTest.java
new file mode 100644
index 0000000..93075a1
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexDeleteTest.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+
+@SuppressWarnings("rawtypes")
+public abstract class OrderedIndexDeleteTest extends OrderedIndexTestDriver {
+
+ private final OrderedIndexTestUtils orderedIndexTestUtils;
+
+ public OrderedIndexDeleteTest(BTreeLeafFrameType[] leafFrameTypesToTest) {
+ super(leafFrameTypesToTest);
+ this.orderedIndexTestUtils = new OrderedIndexTestUtils();
+ }
+
+ private static final int numInsertRounds = 3;
+ private static final int numDeleteRounds = 3;
+
+ @Override
+ protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType,
+ ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey)
+ throws Exception {
+ OrderedIndexTestContext ctx = createTestContext(fieldSerdes, numKeys, leafType);
+ for (int i = 0; i < numInsertRounds; i++) {
+ // We assume all fieldSerdes are of the same type. Check the first
+ // one to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ orderedIndexTestUtils.insertIntTuples(ctx, numTuplesToInsert, getRandom());
+ } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
+ orderedIndexTestUtils.insertStringTuples(ctx, numTuplesToInsert, getRandom());
+ }
+ int numTuplesPerDeleteRound = (int) Math
+ .ceil((float) ctx.getCheckTuples().size() / (float) numDeleteRounds);
+ for (int j = 0; j < numDeleteRounds; j++) {
+ orderedIndexTestUtils.deleteTuples(ctx, numTuplesPerDeleteRound, getRandom());
+ orderedIndexTestUtils.checkPointSearches(ctx);
+ orderedIndexTestUtils.checkScan(ctx);
+ orderedIndexTestUtils.checkDiskOrderScan(ctx);
+ orderedIndexTestUtils.checkRangeSearch(ctx, lowKey, highKey, true, true);
+ if (prefixLowKey != null && prefixHighKey != null) {
+ orderedIndexTestUtils.checkRangeSearch(ctx, prefixLowKey, prefixHighKey, true, true);
+ }
+ }
+ }
+ ctx.getIndex().close();
+ }
+
+ @Override
+ protected String getTestOpName() {
+ return "Delete";
+ }
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexExamplesTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
new file mode 100644
index 0000000..a29be89
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
@@ -0,0 +1,644 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.util.Random;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+@SuppressWarnings("rawtypes")
+public abstract class OrderedIndexExamplesTest {
+ protected static final Logger LOGGER = Logger.getLogger(OrderedIndexExamplesTest.class.getName());
+ protected final Random rnd = new Random(50);
+
+ protected abstract ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories)
+ throws TreeIndexException;
+
+ protected abstract int getIndexFileId();
+
+ /**
+ * Fixed-Length Key,Value Example.
+ *
+ * Create a tree index with one fixed-length key field and one fixed-length value
+ * field. Fill index with random values using insertions (not bulk load).
+ * Perform scans and range search.
+ */
+ @Test
+ public void fixedLengthKeyValueExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Fixed-Length Key,Value Example.");
+ }
+
+ // Declare fields.
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE };
+
+ // Declare keys.
+ int keyFieldCount = 1;
+ IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
+ cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ int indexFileId = getIndexFileId();
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories);
+ treeIndex.create(indexFileId);
+ treeIndex.open(indexFileId);
+
+ long start = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inserting into tree...");
+ }
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor();
+ int numInserts = 10000;
+ for (int i = 0; i < numInserts; i++) {
+ int f0 = rnd.nextInt() % numInserts;
+ int f1 = 5;
+ TupleUtils.createIntegerTuple(tb, tuple, f0, f1);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + i + " : " + f0 + " " + f1);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
+ }
+
+ orderedScan(indexAccessor, fieldSerdes);
+ diskOrderScan(indexAccessor, fieldSerdes);
+
+ // Build low key.
+ ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(keyFieldCount);
+ ArrayTupleReference lowKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(lowKeyTb, lowKey, -1000);
+
+ // Build high key.
+ ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(keyFieldCount);
+ ArrayTupleReference highKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(highKeyTb, highKey, 1000);
+
+ rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey);
+
+ treeIndex.close();
+ }
+
+ /**
+ * Composite Key Example (Non-Unique Index).
+ *
+ * Create a tree index with two fixed-length key fields and one fixed-length
+ * value field. Fill index with random values using insertions (not bulk
+ * load) Perform scans and range search.
+ */
+ @Test
+ public void twoFixedLengthKeysOneFixedLengthValueExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Composite Key Test");
+ }
+
+ // Declare fields.
+ int fieldCount = 3;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+
+ // declare keys
+ int keyFieldCount = 2;
+ IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
+ cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ int indexFileId = getIndexFileId();
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories);
+ treeIndex.create(indexFileId);
+ treeIndex.open(indexFileId);
+
+ long start = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inserting into tree...");
+ }
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor();
+ int numInserts = 10000;
+ for (int i = 0; i < 10000; i++) {
+ int f0 = rnd.nextInt() % 2000;
+ int f1 = rnd.nextInt() % 1000;
+ int f2 = 5;
+ TupleUtils.createIntegerTuple(tb, tuple, f0, f1, f2);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + i + " : " + f0 + " " + f1 + " " + f2);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
+ }
+
+ orderedScan(indexAccessor, fieldSerdes);
+ diskOrderScan(indexAccessor, fieldSerdes);
+
+ // Build low key.
+ ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference lowKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(lowKeyTb, lowKey, -3);
+
+ // Build high key.
+ ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference highKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(highKeyTb, highKey, 3);
+
+ // Prefix-Range search in [-3, 3]
+ rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey);
+
+ treeIndex.close();
+ }
+
+ /**
+ * Variable-Length Example. Create a BTree with one variable-length key
+ * field and one variable-length value field. Fill BTree with random values
+ * using insertions (not bulk load) Perform ordered scans and range search.
+ */
+ @Test
+ public void varLenKeyValueExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Variable-Length Key,Value Example");
+ }
+
+ // Declare fields.
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+
+ // Declare keys.
+ int keyFieldCount = 1;
+ IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
+ cmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+
+ int indexFileId = getIndexFileId();
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories);
+ treeIndex.create(indexFileId);
+ treeIndex.open(indexFileId);
+
+ long start = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inserting into tree...");
+ }
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor();
+ // Max string length to be generated.
+ int maxLength = 10;
+ int numInserts = 10000;
+ for (int i = 0; i < 10000; i++) {
+ String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + f0 + " " + f1);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
+ }
+
+ orderedScan(indexAccessor, fieldSerdes);
+ diskOrderScan(indexAccessor, fieldSerdes);
+
+ // Build low key.
+ ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference lowKey = new ArrayTupleReference();
+ TupleUtils.createTuple(lowKeyTb, lowKey, fieldSerdes, "cbf");
+
+ // Build high key.
+ ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference highKey = new ArrayTupleReference();
+ TupleUtils.createTuple(highKeyTb, highKey, fieldSerdes, "cc7");
+
+ rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey);
+
+ treeIndex.close();
+ }
+
+ /**
+ * Deletion Example.
+ *
+ * Create a BTree with one variable-length key field and one variable-length
+ * value field. Fill B-tree with random values using insertions, then delete
+ * entries one-by-one. Repeat procedure a few times on same BTree.
+ */
+ @Test
+ public void deleteExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Deletion Example");
+ }
+
+ // Declare fields.
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+
+ // Declare keys.
+ int keyFieldCount = 1;
+ IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
+ cmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+
+ int indexFileId = getIndexFileId();
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories);
+ treeIndex.create(indexFileId);
+ treeIndex.open(indexFileId);
+
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor();
+ // Max string length to be generated.
+ int runs = 3;
+ for (int run = 0; run < runs; run++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Deletion example run: " + (run + 1) + "/" + runs);
+ LOGGER.info("Inserting into tree...");
+ }
+ int maxLength = 10;
+ int ins = 10000;
+ String[] f0s = new String[ins];
+ String[] f1s = new String[ins];
+ int insDone = 0;
+ int[] insDoneCmp = new int[ins];
+ for (int i = 0; i < ins; i++) {
+ String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
+ f0s[i] = f0;
+ f1s[i] = f1;
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + i);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ insDone++;
+ } catch (TreeIndexException e) {
+ }
+ insDoneCmp[i] = insDone;
+ }
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Deleting from tree...");
+ }
+ int delDone = 0;
+ for (int i = 0; i < ins; i++) {
+ TupleUtils.createTuple(tb, tuple, fieldSerdes, f0s[i], f1s[i]);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Deleting " + i);
+ }
+ }
+ try {
+ indexAccessor.delete(tuple);
+ delDone++;
+ } catch (TreeIndexException e) {
+ }
+ if (insDoneCmp[i] != delDone) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("INCONSISTENT STATE, ERROR IN DELETION EXAMPLE.");
+ LOGGER.info("INSDONECMP: " + insDoneCmp[i] + " " + delDone);
+ }
+ break;
+ }
+ }
+ if (insDone != delDone) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("ERROR! INSDONE: " + insDone + " DELDONE: " + delDone);
+ }
+ break;
+ }
+ }
+ treeIndex.close();
+ }
+
+ /**
+ * Update example.
+ *
+ * Create a BTree with one variable-length key field and one variable-length
+ * value field. Fill B-tree with random values using insertions, then update
+ * entries one-by-one. Repeat procedure a few times on same BTree.
+ */
+ @Test
+ public void updateExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Update example");
+ }
+
+ // Declare fields.
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+
+ // Declare keys.
+ int keyFieldCount = 1;
+ IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
+ cmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+
+ int indexFileId = getIndexFileId();
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories);
+ treeIndex.create(indexFileId);
+ treeIndex.open(indexFileId);
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inserting into tree...");
+ }
+ IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor();
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ int maxLength = 10;
+ int ins = 10000;
+ String[] keys = new String[10000];
+ for (int i = 0; i < ins; i++) {
+ String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
+ keys[i] = f0;
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + i);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ // Print before doing any updates.
+ orderedScan(indexAccessor, fieldSerdes);
+
+ int runs = 3;
+ for (int run = 0; run < runs; run++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Update test run: " + (run + 1) + "/" + runs);
+ LOGGER.info("Updating BTree");
+ }
+ for (int i = 0; i < ins; i++) {
+ // Generate a new random value for f1.
+ String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ TupleUtils.createTuple(tb, tuple, fieldSerdes, keys[i], f1);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Updating " + i);
+ }
+ }
+ try {
+ indexAccessor.update(tuple);
+ } catch (TreeIndexException e) {
+ } catch (UnsupportedOperationException e) {
+ }
+ }
+ // Do another scan after a round of updates.
+ orderedScan(indexAccessor, fieldSerdes);
+ }
+ treeIndex.close();
+ }
+
+ /**
+ * Bulk load example.
+ *
+ * Load a tree with 100,000 tuples. BTree has a composite key to "simulate"
+ * non-unique index creation.
+ *
+ */
+ @Test
+ public void bulkLoadExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Bulk load example");
+ }
+ // Declare fields.
+ int fieldCount = 3;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+
+ // declare keys
+ int keyFieldCount = 2;
+ IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
+ cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ int indexFileId = getIndexFileId();
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories);
+ treeIndex.create(indexFileId);
+ treeIndex.open(indexFileId);
+
+ // Load sorted records.
+ int ins = 100000;
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Bulk loading " + ins + " tuples");
+ }
+ long start = System.currentTimeMillis();
+ IIndexBulkLoadContext bulkLoadCtx = treeIndex.beginBulkLoad(0.7f);
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ for (int i = 0; i < ins; i++) {
+ TupleUtils.createIntegerTuple(tb, tuple, i, i, 5);
+ treeIndex.bulkLoadAddTuple(tuple, bulkLoadCtx);
+ }
+ treeIndex.endBulkLoad(bulkLoadCtx);
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(ins + " tuples loaded in " + (end - start) + "ms");
+ }
+
+ IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor();
+
+ // Build low key.
+ ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference lowKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(lowKeyTb, lowKey, 44444);
+
+ // Build high key.
+ ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference highKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(highKeyTb, highKey, 44500);
+
+ // Prefix-Range search in [44444, 44500]
+ rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey);
+
+ treeIndex.close();
+ }
+
+ private void orderedScan(IIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes)
+ throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Ordered Scan:");
+ }
+ IIndexCursor scanCursor = (IIndexCursor) indexAccessor.createSearchCursor();
+ RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null);
+ indexAccessor.search(scanCursor, nullPred);
+ try {
+ while (scanCursor.hasNext()) {
+ scanCursor.next();
+ ITupleReference frameTuple = scanCursor.getTuple();
+ String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rec);
+ }
+ }
+ } finally {
+ scanCursor.close();
+ }
+ }
+
+ private void diskOrderScan(IIndexAccessor indexAccessor,
+ ISerializerDeserializer[] fieldSerdes) throws Exception {
+ try {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Disk-Order Scan:");
+ }
+ ITreeIndexAccessor treeIndexAccessor = (ITreeIndexAccessor) indexAccessor;
+ TreeDiskOrderScanCursor diskOrderCursor = (TreeDiskOrderScanCursor) treeIndexAccessor
+ .createDiskOrderScanCursor();
+ treeIndexAccessor.diskOrderScan(diskOrderCursor);
+ try {
+ while (diskOrderCursor.hasNext()) {
+ diskOrderCursor.next();
+ ITupleReference frameTuple = diskOrderCursor.getTuple();
+ String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rec);
+ }
+ }
+ } finally {
+ diskOrderCursor.close();
+ }
+ } catch (UnsupportedOperationException e) {
+ // Ignore exception because some indexes, e.g. the LSMBTree, don't
+ // support disk-order scan.
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Ignoring disk-order scan since it's not supported.");
+ }
+ } catch (ClassCastException e) {
+ // Ignore exception because IIndexAccessor sometimes isn't
+ // an ITreeIndexAccessor, e.g., for the LSMBTree.
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Ignoring disk-order scan since it's not supported.");
+ }
+ }
+ }
+
+ private void rangeSearch(IBinaryComparatorFactory[] cmpFactories, IIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes,
+ ITupleReference lowKey, ITupleReference highKey) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ String lowKeyString = TupleUtils.printTuple(lowKey, fieldSerdes);
+ String highKeyString = TupleUtils.printTuple(highKey, fieldSerdes);
+ LOGGER.info("Range-Search in: [ " + lowKeyString + ", " + highKeyString + "]");
+ }
+ ITreeIndexCursor rangeCursor = (ITreeIndexCursor) indexAccessor.createSearchCursor();
+ MultiComparator lowKeySearchCmp = BTreeUtils.getSearchMultiComparator(cmpFactories, lowKey);
+ MultiComparator highKeySearchCmp = BTreeUtils.getSearchMultiComparator(cmpFactories, highKey);
+ RangePredicate rangePred = new RangePredicate(lowKey, highKey, true, true, lowKeySearchCmp,
+ highKeySearchCmp);
+ indexAccessor.search(rangeCursor, rangePred);
+ try {
+ while (rangeCursor.hasNext()) {
+ rangeCursor.next();
+ ITupleReference frameTuple = rangeCursor.getTuple();
+ String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rec);
+ }
+ }
+ } finally {
+ rangeCursor.close();
+ }
+ }
+
+ public static String randomString(int length, Random random) {
+ String s = Long.toHexString(Double.doubleToLongBits(random.nextDouble()));
+ StringBuilder strBuilder = new StringBuilder();
+ for (int i = 0; i < s.length() && i < length; i++) {
+ strBuilder.append(s.charAt(Math.abs(random.nextInt()) % s.length()));
+ }
+ return strBuilder.toString();
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexInsertTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexInsertTest.java
new file mode 100644
index 0000000..d12603b
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexInsertTest.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+
+/**
+ * Tests the BTree insert operation with strings and integer fields using
+ * various numbers of key and payload fields.
+ *
+ * Each tests first fills a BTree with randomly generated tuples. We compare the
+ * following operations against expected results: 1. Point searches for all
+ * tuples. 2. Ordered scan. 3. Disk-order scan. 4. Range search (and prefix
+ * search for composite keys).
+ *
+ */
+@SuppressWarnings("rawtypes")
+public abstract class OrderedIndexInsertTest extends OrderedIndexTestDriver {
+
+ private final OrderedIndexTestUtils orderedIndexTestUtils;
+
+ public OrderedIndexInsertTest(BTreeLeafFrameType[] leafFrameTypesToTest) {
+ super(leafFrameTypesToTest);
+ this.orderedIndexTestUtils = new OrderedIndexTestUtils();
+ }
+
+ @Override
+ protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType,
+ ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey)
+ throws Exception {
+ OrderedIndexTestContext ctx = createTestContext(fieldSerdes, numKeys, leafType);
+ // We assume all fieldSerdes are of the same type. Check the first one
+ // to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ orderedIndexTestUtils.insertIntTuples(ctx, numTuplesToInsert, getRandom());
+ } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
+ orderedIndexTestUtils.insertStringTuples(ctx, numTuplesToInsert, getRandom());
+ }
+
+ orderedIndexTestUtils.checkPointSearches(ctx);
+ orderedIndexTestUtils.checkScan(ctx);
+ orderedIndexTestUtils.checkDiskOrderScan(ctx);
+
+ orderedIndexTestUtils.checkRangeSearch(ctx, lowKey, highKey, true, true);
+ if (prefixLowKey != null && prefixHighKey != null) {
+ orderedIndexTestUtils.checkRangeSearch(ctx, prefixLowKey, prefixHighKey, true, true);
+ }
+ ctx.getIndex().close();
+ }
+
+ @Override
+ protected String getTestOpName() {
+ return "Insert";
+ }
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexMultiThreadTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexMultiThreadTest.java
new file mode 100644
index 0000000..3a894a2
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexMultiThreadTest.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.util.ArrayList;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
+import edu.uci.ics.hyracks.storage.am.common.ITreeIndexTestWorkerFactory;
+import edu.uci.ics.hyracks.storage.am.common.TestWorkloadConf;
+import edu.uci.ics.hyracks.storage.am.common.TreeIndexMultiThreadTestDriver;
+import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+
+@SuppressWarnings("rawtypes")
+public abstract class OrderedIndexMultiThreadTest {
+
+ protected final Logger LOGGER = Logger.getLogger(OrderedIndexMultiThreadTest.class.getName());
+
+ // Machine-specific number of threads to use for testing.
+ protected final int REGULAR_NUM_THREADS = Runtime.getRuntime().availableProcessors();
+ // Excessive number of threads for testing.
+ protected final int EXCESSIVE_NUM_THREADS = Runtime.getRuntime().availableProcessors() * 4;
+ protected final int NUM_OPERATIONS = 10000;
+
+ protected ArrayList<TestWorkloadConf> workloadConfs = getTestWorkloadConf();
+
+ protected abstract void setUp() throws HyracksException;
+
+ protected abstract void tearDown() throws HyracksDataException;
+
+ protected abstract ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories) throws TreeIndexException;
+
+ protected abstract int getFileId();
+
+ protected abstract ITreeIndexTestWorkerFactory getWorkerFactory();
+
+ protected abstract ArrayList<TestWorkloadConf> getTestWorkloadConf();
+
+ protected abstract String getIndexTypeName();
+
+ protected static float[] getUniformOpProbs(TestOperation[] ops) {
+ float[] opProbs = new float[ops.length];
+ for (int i = 0; i < ops.length; i++) {
+ opProbs[i] = 1.0f / (float) ops.length;
+ }
+ return opProbs;
+ }
+
+ protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, int numThreads, TestWorkloadConf conf, String dataMsg) throws InterruptedException, TreeIndexException, HyracksException {
+ setUp();
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ String indexTypeName = getIndexTypeName();
+ LOGGER.info(indexTypeName + " MultiThread Test:\nData: " + dataMsg + "; Threads: " + numThreads + "; Workload: " + conf.toString() + ".");
+ }
+
+ ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
+ IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeys);
+
+ ITreeIndex index = createTreeIndex(typeTraits, cmpFactories);
+ ITreeIndexTestWorkerFactory workerFactory = getWorkerFactory();
+
+ // 4 batches per thread.
+ int batchSize = (NUM_OPERATIONS / numThreads) / 4;
+
+ TreeIndexMultiThreadTestDriver driver = new TreeIndexMultiThreadTestDriver(index, workerFactory, fieldSerdes, conf.ops, conf.opProbs);
+ driver.init(getFileId());
+ long[] times = driver.run(numThreads, 1, NUM_OPERATIONS, batchSize);
+ driver.deinit();
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("BTree MultiThread Test Time: " + times[0] + "ms");
+ }
+
+ tearDown();
+ }
+
+ @Test
+ public void oneIntKeyAndValue() throws InterruptedException, TreeIndexException, HyracksException {
+ ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+ int numKeys = 1;
+ String dataMsg = "One Int Key And Value";
+
+ for (TestWorkloadConf conf : workloadConfs) {
+ runTest(fieldSerdes, numKeys, REGULAR_NUM_THREADS, conf, dataMsg);
+ runTest(fieldSerdes, numKeys, EXCESSIVE_NUM_THREADS, conf, dataMsg);
+ }
+ }
+
+ @Test
+ public void oneStringKeyAndValue() throws InterruptedException, TreeIndexException, HyracksException {
+ ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE };
+ int numKeys = 1;
+ String dataMsg = "One String Key And Value";
+
+ for (TestWorkloadConf conf : workloadConfs) {
+ runTest(fieldSerdes, numKeys, REGULAR_NUM_THREADS, conf, dataMsg);
+ runTest(fieldSerdes, numKeys, EXCESSIVE_NUM_THREADS, conf, dataMsg);
+ }
+ }
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexTestContext.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexTestContext.java
new file mode 100644
index 0000000..f75a1f1
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexTestContext.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.util.Collection;
+import java.util.TreeSet;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.common.CheckTuple;
+import edu.uci.ics.hyracks.storage.am.common.TreeIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+
+@SuppressWarnings("rawtypes")
+public abstract class OrderedIndexTestContext extends TreeIndexTestContext<CheckTuple> {
+
+ protected final TreeSet<CheckTuple> checkTuples = new TreeSet<CheckTuple>();
+
+ public OrderedIndexTestContext(ISerializerDeserializer[] fieldSerdes, ITreeIndex treeIndex) {
+ super(fieldSerdes, treeIndex);
+ }
+
+ public void upsertCheckTuple(CheckTuple checkTuple, Collection<CheckTuple> checkTuples) {
+ if (checkTuples.contains(checkTuple)) {
+ checkTuples.remove(checkTuple);
+ }
+ checkTuples.add(checkTuple);
+ }
+
+ @Override
+ public TreeSet<CheckTuple> getCheckTuples() {
+ return checkTuples;
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexTestDriver.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexTestDriver.java
new file mode 100644
index 0000000..8daa5e0
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexTestDriver.java
@@ -0,0 +1,178 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.util.Random;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+
+@SuppressWarnings("rawtypes")
+public abstract class OrderedIndexTestDriver {
+ protected final Logger LOGGER = Logger.getLogger(OrderedIndexTestDriver.class.getName());
+
+ protected static final int numTuplesToInsert = 10000;
+
+ protected abstract OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
+ BTreeLeafFrameType leafType) throws Exception;
+
+ protected abstract Random getRandom();
+
+ protected abstract void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType,
+ ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey)
+ throws Exception;
+
+ protected abstract String getTestOpName();
+
+ protected final BTreeLeafFrameType[] leafFrameTypesToTest;
+
+ public OrderedIndexTestDriver(BTreeLeafFrameType[] leafFrameTypesToTest) {
+ this.leafFrameTypesToTest = leafFrameTypesToTest;
+ }
+
+ @Test
+ public void oneIntKeyAndValue() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("BTree " + getTestOpName() + " Test With One Int Key And Value.");
+ }
+
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE };
+ // Range search in [-1000, 1000]
+ ITupleReference lowKey = TupleUtils.createIntegerTuple(-1000);
+ ITupleReference highKey = TupleUtils.createIntegerTuple(1000);
+
+ for (BTreeLeafFrameType leafFrameType : leafFrameTypesToTest) {
+ runTest(fieldSerdes, 1, leafFrameType, lowKey, highKey, null, null);
+ }
+ }
+
+ @Test
+ public void twoIntKeys() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("BTree " + getTestOpName() + " Test With Two Int Keys.");
+ }
+
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE };
+
+ // Range search in [50 0, 50 500]
+ ITupleReference lowKey = TupleUtils.createIntegerTuple(50, 0);
+ ITupleReference highKey = TupleUtils.createIntegerTuple(50, 500);
+
+ // Prefix range search in [50, 50]
+ ITupleReference prefixLowKey = TupleUtils.createIntegerTuple(50);
+ ITupleReference prefixHighKey = TupleUtils.createIntegerTuple(50);
+
+ for (BTreeLeafFrameType leafFrameType : leafFrameTypesToTest) {
+ runTest(fieldSerdes, 2, leafFrameType, lowKey, highKey, prefixLowKey, prefixHighKey);
+ }
+ }
+
+ @Test
+ public void twoIntKeysAndValues() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("BTree " + getTestOpName() + " Test With Two Int Keys And Values.");
+ }
+
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE };
+
+ // Range search in [50 100, 100 100]
+ ITupleReference lowKey = TupleUtils.createIntegerTuple(-100, -100);
+ ITupleReference highKey = TupleUtils.createIntegerTuple(100, 100);
+
+ // Prefix range search in [50, 50]
+ ITupleReference prefixLowKey = TupleUtils.createIntegerTuple(50);
+ ITupleReference prefixHighKey = TupleUtils.createIntegerTuple(50);
+
+ for (BTreeLeafFrameType leafFrameType : leafFrameTypesToTest) {
+ runTest(fieldSerdes, 2, leafFrameType, lowKey, highKey, prefixLowKey, prefixHighKey);
+ }
+ }
+
+ @Test
+ public void oneStringKeyAndValue() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("BTree " + getTestOpName() + " Test With One String Key And Value.");
+ }
+
+ ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+
+ // Range search in ["cbf", cc7"]
+ ITupleReference lowKey = TupleUtils.createTuple(fieldSerdes, "cbf");
+ ITupleReference highKey = TupleUtils.createTuple(fieldSerdes, "cc7");
+
+ for (BTreeLeafFrameType leafFrameType : leafFrameTypesToTest) {
+ runTest(fieldSerdes, 1, leafFrameType, lowKey, highKey, null, null);
+ }
+ }
+
+ @Test
+ public void twoStringKeys() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("BTree " + getTestOpName() + " Test With Two String Keys.");
+ }
+
+ ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+
+ // Range search in ["cbf", "ddd", cc7", "eee"]
+ ITupleReference lowKey = TupleUtils.createTuple(fieldSerdes, "cbf", "ddd");
+ ITupleReference highKey = TupleUtils.createTuple(fieldSerdes, "cc7", "eee");
+
+ // Prefix range search in ["cbf", cc7"]
+ ITupleReference prefixLowKey = TupleUtils.createTuple(fieldSerdes, "cbf");
+ ITupleReference prefixHighKey = TupleUtils.createTuple(fieldSerdes, "cc7");
+
+ for (BTreeLeafFrameType leafFrameType : leafFrameTypesToTest) {
+ runTest(fieldSerdes, 2, leafFrameType, lowKey, highKey, prefixLowKey, prefixHighKey);
+ }
+ }
+
+ @Test
+ public void twoStringKeysAndValues() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("BTree " + getTestOpName() + " Test With Two String Keys And Values.");
+ }
+
+ ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+
+ // Range search in ["cbf", "ddd", cc7", "eee"]
+ ITupleReference lowKey = TupleUtils.createTuple(fieldSerdes, "cbf", "ddd");
+ ITupleReference highKey = TupleUtils.createTuple(fieldSerdes, "cc7", "eee");
+
+ // Prefix range search in ["cbf", cc7"]
+ ITupleReference prefixLowKey = TupleUtils.createTuple(fieldSerdes, "cbf");
+ ITupleReference prefixHighKey = TupleUtils.createTuple(fieldSerdes, "cc7");
+
+ for (BTreeLeafFrameType leafFrameType : leafFrameTypesToTest) {
+ runTest(fieldSerdes, 2, leafFrameType, lowKey, highKey, prefixLowKey, prefixHighKey);
+ }
+ }
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexTestUtils.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexTestUtils.java
new file mode 100644
index 0000000..a053dde
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexTestUtils.java
@@ -0,0 +1,440 @@
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.NavigableSet;
+import java.util.Random;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeDuplicateKeyException;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
+import edu.uci.ics.hyracks.storage.am.common.CheckTuple;
+import edu.uci.ics.hyracks.storage.am.common.ITreeIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.common.TreeIndexTestUtils;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+@SuppressWarnings("rawtypes")
+public class OrderedIndexTestUtils extends TreeIndexTestUtils {
+ private static final Logger LOGGER = Logger.getLogger(OrderedIndexTestUtils.class.getName());
+
+ private static void compareActualAndExpected(ITupleReference actual, CheckTuple expected,
+ ISerializerDeserializer[] fieldSerdes) throws HyracksDataException {
+ for (int i = 0; i < fieldSerdes.length; i++) {
+ ByteArrayInputStream inStream = new ByteArrayInputStream(actual.getFieldData(i), actual.getFieldStart(i),
+ actual.getFieldLength(i));
+ DataInput dataIn = new DataInputStream(inStream);
+ Object actualObj = fieldSerdes[i].deserialize(dataIn);
+ if (!actualObj.equals(expected.get(i))) {
+ fail("Actual and expected fields do not match on field " + i + ".\nExpected: " + expected.get(i)
+ + "\nActual : " + actualObj);
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ // Create a new TreeSet containing the elements satisfying the prefix
+ // search.
+ // Implementing prefix search by changing compareTo() in CheckTuple does not
+ // work.
+ public static TreeSet<CheckTuple> getPrefixExpectedSubset(TreeSet<CheckTuple> checkTuples, CheckTuple lowKey,
+ CheckTuple highKey) {
+ TreeSet<CheckTuple> expectedSubset = new TreeSet<CheckTuple>();
+ Iterator<CheckTuple> iter = checkTuples.iterator();
+ while (iter.hasNext()) {
+ CheckTuple t = iter.next();
+ boolean geLowKey = true;
+ boolean leHighKey = true;
+ for (int i = 0; i < lowKey.getNumKeys(); i++) {
+ if (t.get(i).compareTo(lowKey.get(i)) < 0) {
+ geLowKey = false;
+ break;
+ }
+ }
+ for (int i = 0; i < highKey.getNumKeys(); i++) {
+ if (t.get(i).compareTo(highKey.get(i)) > 0) {
+ leHighKey = false;
+ break;
+ }
+ }
+ if (geLowKey && leHighKey) {
+ expectedSubset.add(t);
+ }
+ }
+ return expectedSubset;
+ }
+
+ @SuppressWarnings("unchecked")
+ public void checkRangeSearch(ITreeIndexTestContext ctx, ITupleReference lowKey, ITupleReference highKey,
+ boolean lowKeyInclusive, boolean highKeyInclusive) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Testing Range Search.");
+ }
+ MultiComparator lowKeyCmp = BTreeUtils.getSearchMultiComparator(ctx.getComparatorFactories(), lowKey);
+ MultiComparator highKeyCmp = BTreeUtils.getSearchMultiComparator(ctx.getComparatorFactories(), highKey);
+ IIndexCursor searchCursor = ctx.getIndexAccessor().createSearchCursor();
+ RangePredicate rangePred = new RangePredicate(lowKey, highKey, lowKeyInclusive, highKeyInclusive, lowKeyCmp,
+ highKeyCmp);
+ ctx.getIndexAccessor().search(searchCursor, rangePred);
+ // Get the subset of elements from the expected set within given key
+ // range.
+ CheckTuple lowKeyCheck = createCheckTupleFromTuple(lowKey, ctx.getFieldSerdes(), lowKeyCmp.getKeyFieldCount());
+ CheckTuple highKeyCheck = createCheckTupleFromTuple(highKey, ctx.getFieldSerdes(),
+ highKeyCmp.getKeyFieldCount());
+ NavigableSet<CheckTuple> expectedSubset = null;
+ if (lowKeyCmp.getKeyFieldCount() < ctx.getKeyFieldCount()
+ || highKeyCmp.getKeyFieldCount() < ctx.getKeyFieldCount()) {
+ // Searching on a key prefix (low key or high key or both).
+ expectedSubset = getPrefixExpectedSubset((TreeSet<CheckTuple>) ctx.getCheckTuples(), lowKeyCheck,
+ highKeyCheck);
+ } else {
+ // Searching on all key fields.
+ expectedSubset = ((TreeSet<CheckTuple>) ctx.getCheckTuples()).subSet(lowKeyCheck, lowKeyInclusive,
+ highKeyCheck, highKeyInclusive);
+ }
+ Iterator<CheckTuple> checkIter = expectedSubset.iterator();
+ int actualCount = 0;
+ try {
+ while (searchCursor.hasNext()) {
+ if (!checkIter.hasNext()) {
+ fail("Range search returned more answers than expected.\nExpected: " + expectedSubset.size());
+ }
+ searchCursor.next();
+ CheckTuple expectedTuple = checkIter.next();
+ ITupleReference tuple = searchCursor.getTuple();
+ compareActualAndExpected(tuple, expectedTuple, ctx.getFieldSerdes());
+ actualCount++;
+ }
+ if (actualCount < expectedSubset.size()) {
+ fail("Range search returned fewer answers than expected.\nExpected: " + expectedSubset.size()
+ + "\nActual : " + actualCount);
+ }
+ } finally {
+ searchCursor.close();
+ }
+ }
+
+ public void checkPointSearches(ITreeIndexTestContext ictx) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Testing Point Searches On All Expected Keys.");
+ }
+ OrderedIndexTestContext ctx = (OrderedIndexTestContext) ictx;
+ IIndexCursor searchCursor = ctx.getIndexAccessor().createSearchCursor();
+
+ ArrayTupleBuilder lowKeyBuilder = new ArrayTupleBuilder(ctx.getKeyFieldCount());
+ ArrayTupleReference lowKey = new ArrayTupleReference();
+ ArrayTupleBuilder highKeyBuilder = new ArrayTupleBuilder(ctx.getKeyFieldCount());
+ ArrayTupleReference highKey = new ArrayTupleReference();
+ RangePredicate rangePred = new RangePredicate(lowKey, highKey, true, true, null, null);
+
+ // Iterate through expected tuples, and perform a point search in the
+ // BTree to verify the tuple can be reached.
+ for (CheckTuple checkTuple : ctx.getCheckTuples()) {
+ createTupleFromCheckTuple(checkTuple, lowKeyBuilder, lowKey, ctx.getFieldSerdes());
+ createTupleFromCheckTuple(checkTuple, highKeyBuilder, highKey, ctx.getFieldSerdes());
+ MultiComparator lowKeyCmp = BTreeUtils.getSearchMultiComparator(ctx.getComparatorFactories(), lowKey);
+ MultiComparator highKeyCmp = BTreeUtils.getSearchMultiComparator(ctx.getComparatorFactories(), highKey);
+
+ rangePred.setLowKey(lowKey, true);
+ rangePred.setHighKey(highKey, true);
+ rangePred.setLowKeyComparator(lowKeyCmp);
+ rangePred.setHighKeyComparator(highKeyCmp);
+
+ ctx.getIndexAccessor().search(searchCursor, rangePred);
+
+ try {
+ // We expect exactly one answer.
+ if (searchCursor.hasNext()) {
+ searchCursor.next();
+ ITupleReference tuple = searchCursor.getTuple();
+ compareActualAndExpected(tuple, checkTuple, ctx.getFieldSerdes());
+ }
+ if (searchCursor.hasNext()) {
+ fail("Point search returned more than one answer.");
+ }
+ } finally {
+ searchCursor.close();
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public void insertStringTuples(ITreeIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = ctx.getFieldCount();
+ int numKeyFields = ctx.getKeyFieldCount();
+ String[] fieldValues = new String[fieldCount];
+ for (int i = 0; i < numTuples; i++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
+ LOGGER.info("Inserting Tuple " + (i + 1) + "/" + numTuples);
+ }
+ }
+ // Set keys.
+ for (int j = 0; j < numKeyFields; j++) {
+ int length = (Math.abs(rnd.nextInt()) % 10) + 1;
+ fieldValues[j] = getRandomString(length, rnd);
+ }
+ // Set values.
+ for (int j = numKeyFields; j < fieldCount; j++) {
+ fieldValues[j] = getRandomString(5, rnd);
+ }
+ TupleUtils.createTuple(ctx.getTupleBuilder(), ctx.getTuple(), ctx.getFieldSerdes(), (Object[]) fieldValues);
+ try {
+ ctx.getIndexAccessor().insert(ctx.getTuple());
+ // Set expected values. Do this only after insertion succeeds
+ // because we ignore duplicate keys.
+ ctx.insertCheckTuple(createStringCheckTuple(fieldValues, ctx.getKeyFieldCount()), ctx.getCheckTuples());
+ } catch (BTreeDuplicateKeyException e) {
+ // Ignore duplicate key insertions.
+ }
+ }
+ }
+
+ public void upsertStringTuples(ITreeIndexTestContext ictx, int numTuples, Random rnd) throws Exception {
+ OrderedIndexTestContext ctx = (OrderedIndexTestContext) ictx;
+ int fieldCount = ctx.getFieldCount();
+ int numKeyFields = ctx.getKeyFieldCount();
+ String[] fieldValues = new String[fieldCount];
+ for (int i = 0; i < numTuples; i++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
+ LOGGER.info("Inserting Tuple " + (i + 1) + "/" + numTuples);
+ }
+ }
+ // Set keys.
+ for (int j = 0; j < numKeyFields; j++) {
+ int length = (Math.abs(rnd.nextInt()) % 10) + 1;
+ fieldValues[j] = getRandomString(length, rnd);
+ }
+ // Set values.
+ for (int j = numKeyFields; j < fieldCount; j++) {
+ fieldValues[j] = getRandomString(5, rnd);
+ }
+ TupleUtils.createTuple(ctx.getTupleBuilder(), ctx.getTuple(), ctx.getFieldSerdes(), (Object[]) fieldValues);
+ ctx.getIndexAccessor().upsert(ctx.getTuple());
+ ctx.upsertCheckTuple(createStringCheckTuple(fieldValues, ctx.getKeyFieldCount()), ctx.getCheckTuples());
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public void bulkLoadStringTuples(ITreeIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = ctx.getFieldCount();
+ int numKeyFields = ctx.getKeyFieldCount();
+ String[] fieldValues = new String[fieldCount];
+ TreeSet<CheckTuple> tmpCheckTuples = new TreeSet<CheckTuple>();
+ for (int i = 0; i < numTuples; i++) {
+ // Set keys.
+ for (int j = 0; j < numKeyFields; j++) {
+ int length = (Math.abs(rnd.nextInt()) % 10) + 1;
+ fieldValues[j] = getRandomString(length, rnd);
+ }
+ // Set values.
+ for (int j = numKeyFields; j < fieldCount; j++) {
+ fieldValues[j] = getRandomString(5, rnd);
+ }
+ // Set expected values. We also use these as the pre-sorted stream
+ // for bulk loading.
+ ctx.insertCheckTuple(createStringCheckTuple(fieldValues, ctx.getKeyFieldCount()), tmpCheckTuples);
+ }
+ bulkLoadCheckTuples(ctx, tmpCheckTuples);
+
+ // Add tmpCheckTuples to ctx check tuples for comparing searches.
+ for (CheckTuple checkTuple : tmpCheckTuples) {
+ ctx.insertCheckTuple(checkTuple, ctx.getCheckTuples());
+ }
+ }
+
+ public void upsertIntTuples(ITreeIndexTestContext ictx, int numTuples, Random rnd) throws Exception {
+ OrderedIndexTestContext ctx = (OrderedIndexTestContext) ictx;
+ int fieldCount = ctx.getFieldCount();
+ int numKeyFields = ctx.getKeyFieldCount();
+ int[] fieldValues = new int[ctx.getFieldCount()];
+ // Scale range of values according to number of keys.
+ // For example, for 2 keys we want the square root of numTuples, for 3
+ // keys the cube root of numTuples, etc.
+ int maxValue = (int) Math.ceil(Math.pow(numTuples, 1.0 / (double) numKeyFields));
+ for (int i = 0; i < numTuples; i++) {
+ // Set keys.
+ setIntKeyFields(fieldValues, numKeyFields, maxValue, rnd);
+ // Set values.
+ setIntPayloadFields(fieldValues, numKeyFields, fieldCount);
+ TupleUtils.createIntegerTuple(ctx.getTupleBuilder(), ctx.getTuple(), fieldValues);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
+ LOGGER.info("Inserting Tuple " + (i + 1) + "/" + numTuples);
+ }
+ }
+ ctx.getIndexAccessor().upsert(ctx.getTuple());
+ ctx.upsertCheckTuple(createIntCheckTuple(fieldValues, ctx.getKeyFieldCount()), ctx.getCheckTuples());
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public void updateTuples(ITreeIndexTestContext ictx, int numTuples, Random rnd) throws Exception {
+ OrderedIndexTestContext ctx = (OrderedIndexTestContext) ictx;
+ int fieldCount = ctx.getFieldCount();
+ int keyFieldCount = ctx.getKeyFieldCount();
+ // This is a noop because we can only update non-key fields.
+ if (fieldCount == keyFieldCount) {
+ return;
+ }
+ ArrayTupleBuilder updateTupleBuilder = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference updateTuple = new ArrayTupleReference();
+ int numCheckTuples = ctx.getCheckTuples().size();
+ // Copy CheckTuple references into array, so we can randomly pick from
+ // there.
+ CheckTuple[] checkTuples = new CheckTuple[numCheckTuples];
+ int idx = 0;
+ for (CheckTuple checkTuple : ctx.getCheckTuples()) {
+ checkTuples[idx++] = checkTuple;
+ }
+ for (int i = 0; i < numTuples && numCheckTuples > 0; i++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
+ LOGGER.info("Updating Tuple " + (i + 1) + "/" + numTuples);
+ }
+ }
+ int checkTupleIdx = Math.abs(rnd.nextInt() % numCheckTuples);
+ CheckTuple checkTuple = checkTuples[checkTupleIdx];
+ // Update check tuple's non-key fields.
+ for (int j = keyFieldCount; j < fieldCount; j++) {
+ Comparable newValue = getRandomUpdateValue(ctx.getFieldSerdes()[j], rnd);
+ checkTuple.set(j, newValue);
+ }
+
+ createTupleFromCheckTuple(checkTuple, updateTupleBuilder, updateTuple, ctx.getFieldSerdes());
+ ctx.getIndexAccessor().update(updateTuple);
+
+ // Swap with last "valid" CheckTuple.
+ CheckTuple tmp = checkTuples[numCheckTuples - 1];
+ checkTuples[numCheckTuples - 1] = checkTuple;
+ checkTuples[checkTupleIdx] = tmp;
+ numCheckTuples--;
+ }
+ }
+
+ public CheckTuple createStringCheckTuple(String[] fieldValues, int numKeyFields) {
+ CheckTuple<String> checkTuple = new CheckTuple<String>(fieldValues.length, numKeyFields);
+ for (String s : fieldValues) {
+ checkTuple.add((String) s);
+ }
+ return checkTuple;
+ }
+
+ private static Comparable getRandomUpdateValue(ISerializerDeserializer serde, Random rnd) {
+ if (serde instanceof IntegerSerializerDeserializer) {
+ return Integer.valueOf(rnd.nextInt());
+ } else if (serde instanceof UTF8StringSerializerDeserializer) {
+ return getRandomString(10, rnd);
+ }
+ return null;
+ }
+
+ public static String getRandomString(int length, Random rnd) {
+ String s = Long.toHexString(Double.doubleToLongBits(rnd.nextDouble()));
+ StringBuilder strBuilder = new StringBuilder();
+ for (int i = 0; i < s.length() && i < length; i++) {
+ strBuilder.append(s.charAt(Math.abs(rnd.nextInt()) % s.length()));
+ }
+ return strBuilder.toString();
+ }
+
+ @Override
+ protected CheckTuple createCheckTuple(int numFields, int numKeyFields) {
+ return new CheckTuple(numFields, numKeyFields);
+ }
+
+ @Override
+ protected ISearchPredicate createNullSearchPredicate() {
+ return new RangePredicate(null, null, true, true, null, null);
+ }
+
+ @Override
+ public void checkExpectedResults(ITreeIndexCursor cursor, Collection checkTuples,
+ ISerializerDeserializer[] fieldSerdes, int keyFieldCount, Iterator<CheckTuple> checkIter) throws Exception {
+ int actualCount = 0;
+ try {
+ while (cursor.hasNext()) {
+ if (!checkIter.hasNext()) {
+ fail("Ordered scan returned more answers than expected.\nExpected: " + checkTuples.size());
+ }
+ cursor.next();
+ CheckTuple expectedTuple = checkIter.next();
+ ITupleReference tuple = cursor.getTuple();
+ compareActualAndExpected(tuple, expectedTuple, fieldSerdes);
+ actualCount++;
+ }
+ if (actualCount < checkTuples.size()) {
+ fail("Ordered scan returned fewer answers than expected.\nExpected: " + checkTuples.size()
+ + "\nActual : " + actualCount);
+ }
+ } finally {
+ cursor.close();
+ }
+
+ }
+
+ @Override
+ protected CheckTuple createIntCheckTuple(int[] fieldValues, int numKeyFields) {
+ CheckTuple<Integer> checkTuple = new CheckTuple<Integer>(fieldValues.length, numKeyFields);
+ for (int v : fieldValues) {
+ checkTuple.add(v);
+ }
+ return checkTuple;
+ }
+
+ @Override
+ protected void setIntKeyFields(int[] fieldValues, int numKeyFields, int maxValue, Random rnd) {
+ for (int j = 0; j < numKeyFields; j++) {
+ fieldValues[j] = rnd.nextInt() % maxValue;
+ }
+ }
+
+ @Override
+ protected void setIntPayloadFields(int[] fieldValues, int numKeyFields, int numFields) {
+ for (int j = numKeyFields; j < numFields; j++) {
+ fieldValues[j] = j;
+ }
+ }
+
+ @Override
+ protected Collection createCheckTuplesCollection() {
+ return new TreeSet<CheckTuple>();
+ }
+
+ @Override
+ protected ArrayTupleBuilder createDeleteTupleBuilder(ITreeIndexTestContext ctx) {
+ return new ArrayTupleBuilder(ctx.getKeyFieldCount());
+ }
+
+ @Override
+ protected boolean checkDiskOrderScanResult(ITupleReference tuple, CheckTuple checkTuple, ITreeIndexTestContext ctx)
+ throws HyracksDataException {
+ @SuppressWarnings("unchecked")
+ TreeSet<CheckTuple> checkTuples = (TreeSet<CheckTuple>) ctx.getCheckTuples();
+ CheckTuple matchingCheckTuple = checkTuples.floor(checkTuple);
+ if (matchingCheckTuple == null) {
+ return false;
+ }
+ compareActualAndExpected(tuple, matchingCheckTuple, ctx.getFieldSerdes());
+ return true;
+ }
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexUpdateTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexUpdateTest.java
new file mode 100644
index 0000000..65b2ade
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexUpdateTest.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+
+@SuppressWarnings("rawtypes")
+public abstract class OrderedIndexUpdateTest extends OrderedIndexTestDriver {
+
+ private final OrderedIndexTestUtils orderedIndexTestUtils;
+
+ public OrderedIndexUpdateTest(BTreeLeafFrameType[] leafFrameTypesToTest) {
+ super(leafFrameTypesToTest);
+ this.orderedIndexTestUtils = new OrderedIndexTestUtils();
+ }
+
+ private static final int numUpdateRounds = 3;
+
+ @Override
+ protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType,
+ ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey)
+ throws Exception {
+ // This is a noop because we can only update non-key fields.
+ if (fieldSerdes.length == numKeys) {
+ return;
+ }
+ OrderedIndexTestContext ctx = createTestContext(fieldSerdes, numKeys, leafType);
+ // We assume all fieldSerdes are of the same type. Check the first one
+ // to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ orderedIndexTestUtils.insertIntTuples(ctx, numTuplesToInsert, getRandom());
+ } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
+ orderedIndexTestUtils.insertStringTuples(ctx, numTuplesToInsert, getRandom());
+ }
+ int numTuplesPerDeleteRound = (int) Math.ceil((float) ctx.getCheckTuples().size() / (float) numUpdateRounds);
+ for (int j = 0; j < numUpdateRounds; j++) {
+ orderedIndexTestUtils.updateTuples(ctx, numTuplesPerDeleteRound, getRandom());
+ orderedIndexTestUtils.checkPointSearches(ctx);
+ orderedIndexTestUtils.checkScan(ctx);
+ orderedIndexTestUtils.checkDiskOrderScan(ctx);
+ orderedIndexTestUtils.checkRangeSearch(ctx, lowKey, highKey, true, true);
+ if (prefixLowKey != null && prefixHighKey != null) {
+ orderedIndexTestUtils.checkRangeSearch(ctx, prefixLowKey, prefixHighKey, true, true);
+ }
+ }
+ }
+
+ @Override
+ protected String getTestOpName() {
+ return "Update";
+ }
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexUpsertTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexUpsertTest.java
new file mode 100644
index 0000000..0d94a18
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexUpsertTest.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+
+/**
+ * Tests the BTree insert operation with strings and integer fields using
+ * various numbers of key and payload fields.
+ *
+ * Each tests first fills a BTree with randomly generated tuples. We compare the
+ * following operations against expected results: 1. Point searches for all
+ * tuples. 2. Ordered scan. 3. Disk-order scan. 4. Range search (and prefix
+ * search for composite keys).
+ *
+ */
+@SuppressWarnings("rawtypes")
+public abstract class OrderedIndexUpsertTest extends OrderedIndexTestDriver {
+
+ private final OrderedIndexTestUtils orderedIndexTestUtils;
+
+ public OrderedIndexUpsertTest(BTreeLeafFrameType[] leafFrameTypesToTest) {
+ super(leafFrameTypesToTest);
+ this.orderedIndexTestUtils = new OrderedIndexTestUtils();
+ }
+
+ @Override
+ protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType,
+ ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey)
+ throws Exception {
+ OrderedIndexTestContext ctx = createTestContext(fieldSerdes, numKeys, leafType);
+ // We assume all fieldSerdes are of the same type. Check the first one
+ // to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ orderedIndexTestUtils.upsertIntTuples(ctx, numTuplesToInsert, getRandom());
+ } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
+ orderedIndexTestUtils.upsertStringTuples(ctx, numTuplesToInsert, getRandom());
+ }
+
+ orderedIndexTestUtils.checkPointSearches(ctx);
+ orderedIndexTestUtils.checkScan(ctx);
+ orderedIndexTestUtils.checkDiskOrderScan(ctx);
+
+ orderedIndexTestUtils.checkRangeSearch(ctx, lowKey, highKey, true, true);
+ if (prefixLowKey != null && prefixHighKey != null) {
+ orderedIndexTestUtils.checkRangeSearch(ctx, prefixLowKey, prefixHighKey, true, true);
+ }
+ ctx.getIndex().close();
+ }
+
+ @Override
+ protected String getTestOpName() {
+ return "Insert";
+ }
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/AbstractTreeIndexTestWorker.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/AbstractTreeIndexTestWorker.java
new file mode 100644
index 0000000..eca9b35
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/AbstractTreeIndexTestWorker.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common;
+
+import java.util.Random;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
+import edu.uci.ics.hyracks.storage.am.common.datagen.TupleBatch;
+
+public abstract class AbstractTreeIndexTestWorker extends Thread implements ITreeIndexTestWorker {
+ private Random rnd = new Random();
+ private final DataGenThread dataGen;
+ private final TestOperationSelector opSelector;
+ private final int numBatches;
+
+ protected final IIndexAccessor indexAccessor;
+
+ public AbstractTreeIndexTestWorker(DataGenThread dataGen, TestOperationSelector opSelector, ITreeIndex index, int numBatches) {
+ this.dataGen = dataGen;
+ this.opSelector = opSelector;
+ this.numBatches = numBatches;
+ indexAccessor = index.createAccessor();
+ }
+
+ @Override
+ public void run() {
+ try {
+ for (int i = 0; i < numBatches; i++) {
+ TupleBatch batch = dataGen.getBatch();
+ for (int j = 0; j < batch.size(); j++) {
+ TestOperation op = opSelector.getOp(rnd.nextInt());
+ ITupleReference tuple = batch.get(j);
+ performOp(tuple, op);
+ }
+ dataGen.releaseBatch(batch);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ protected void consumeCursorTuples(IIndexCursor cursor) throws HyracksDataException {
+ try {
+ while (cursor.hasNext()) {
+ cursor.next();
+ }
+ } finally {
+ cursor.close();
+ }
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/CheckTuple.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/CheckTuple.java
similarity index 92%
rename from hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/CheckTuple.java
rename to hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/CheckTuple.java
index f945ab9..4b4b90b 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/CheckTuple.java
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/CheckTuple.java
@@ -13,13 +13,13 @@
* limitations under the License.
*/
-package edu.uci.ics.hyracks.storage.am.btree.util;
+package edu.uci.ics.hyracks.storage.am.common;
@SuppressWarnings({"rawtypes", "unchecked"})
public class CheckTuple<T extends Comparable<T>> implements Comparable<T> {
- private final int numKeys;
- private final Comparable[] tuple;
- private int pos;
+ protected final int numKeys;
+ protected final Comparable[] tuple;
+ protected int pos;
public CheckTuple(int numFields, int numKeys) {
this.numKeys = numKeys;
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/ITreeIndexTestContext.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/ITreeIndexTestContext.java
new file mode 100644
index 0000000..9be3e29
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/ITreeIndexTestContext.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common;
+
+import java.util.Collection;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+
+@SuppressWarnings("rawtypes")
+public interface ITreeIndexTestContext<T extends CheckTuple> {
+ public int getFieldCount();
+
+ public int getKeyFieldCount();
+
+ public ISerializerDeserializer[] getFieldSerdes();
+
+ public IBinaryComparatorFactory[] getComparatorFactories();
+
+ public IIndexAccessor getIndexAccessor();
+
+ public ITreeIndex getIndex();
+
+ public ArrayTupleReference getTuple();
+
+ public ArrayTupleBuilder getTupleBuilder();
+
+ public void insertCheckTuple(T checkTuple, Collection<T> checkTuples);
+
+ public void deleteCheckTuple(T checkTuple, Collection<T> checkTuples);
+
+ public Collection<T> getCheckTuples();
+
+}
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexResultCursor.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/ITreeIndexTestWorker.java
similarity index 63%
rename from hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexResultCursor.java
rename to hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/ITreeIndexTestWorker.java
index fb73a65..5e2733f 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexResultCursor.java
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/ITreeIndexTestWorker.java
@@ -13,16 +13,13 @@
* limitations under the License.
*/
-package edu.uci.ics.hyracks.storage.am.invertedindex.api;
+package edu.uci.ics.hyracks.storage.am.common;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
-public interface IInvertedIndexResultCursor {
- public boolean hasNext();
-
- public void next();
-
- public ITupleReference getTuple();
-
- public void reset(IInvertedIndexSearcher invIndexSearcher);
+public interface ITreeIndexTestWorker {
+ void performOp(ITupleReference tuple, TestOperation op) throws HyracksDataException, IndexException;
}
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IReceiverEventListener.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/ITreeIndexTestWorkerFactory.java
similarity index 63%
copy from hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IReceiverEventListener.java
copy to hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/ITreeIndexTestWorkerFactory.java
index 98c083e..64b5aea 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IReceiverEventListener.java
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/ITreeIndexTestWorkerFactory.java
@@ -12,10 +12,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.api.comm;
-public interface IReceiverEventListener {
- public void notifySenderAvailability(int sender);
+package edu.uci.ics.hyracks.storage.am.common;
- public void notifySenderCount(int senderCount);
-}
\ No newline at end of file
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
+
+public interface ITreeIndexTestWorkerFactory {
+ public AbstractTreeIndexTestWorker create(DataGenThread dataGen, TestOperationSelector opSelector, ITreeIndex index, int numBatches);
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TestOperationSelector.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TestOperationSelector.java
new file mode 100644
index 0000000..1ae79a1
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TestOperationSelector.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common;
+
+import java.util.Arrays;
+
+public class TestOperationSelector {
+
+ public static enum TestOperation {
+ INSERT,
+ DELETE,
+ UPDATE,
+ UPSERT,
+ POINT_SEARCH,
+ RANGE_SEARCH,
+ SCAN,
+ DISKORDER_SCAN,
+ MERGE
+ }
+
+ private final TestOperation[] ops;
+ private final int[] opRanges;
+
+ public TestOperationSelector(TestOperation[] ops, float[] opProbs) {
+ sanityCheck(ops, opProbs);
+ this.ops = ops;
+ this.opRanges = getOpRanges(opProbs);
+ }
+
+ private void sanityCheck(TestOperation[] ops, float[] opProbs) {
+ if (ops.length == 0) {
+ throw new RuntimeException("Empty op array.");
+ }
+ if (opProbs.length == 0) {
+ throw new RuntimeException("Empty op probabilities.");
+ }
+ if (ops.length != opProbs.length) {
+ throw new RuntimeException("Ops and op probabilities have unequal length.");
+ }
+ float sum = 0.0f;
+ for (int i = 0; i < opProbs.length; i++) {
+ sum += opProbs[i];
+ }
+ if (sum != 1.0f) {
+ throw new RuntimeException("Op probabilities don't add up to 1.");
+ }
+ }
+
+ private int[] getOpRanges(float[] opProbabilities) {
+ int[] opRanges = new int[opProbabilities.length];
+ if (opRanges.length > 1) {
+ opRanges[0] = (int) Math.floor(Integer.MAX_VALUE * opProbabilities[0]);
+ for (int i = 1; i < opRanges.length - 1; i++) {
+ opRanges[i] = opRanges[i - 1] + (int) Math.floor(Integer.MAX_VALUE * opProbabilities[i]);
+ }
+ opRanges[opRanges.length - 1] = Integer.MAX_VALUE;
+ } else {
+ opRanges[0] = Integer.MAX_VALUE;
+ }
+ return opRanges;
+ }
+
+ public TestOperation getOp(int randomInt) {
+ int ix = Arrays.binarySearch(opRanges, randomInt);
+ if (ix < 0) {
+ ix = -ix - 1;
+ }
+ return ops[ix];
+ }
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TestWorkloadConf.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TestWorkloadConf.java
new file mode 100644
index 0000000..2437514
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TestWorkloadConf.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common;
+
+import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
+
+public class TestWorkloadConf {
+ public final TestOperation[] ops;
+ public final float[] opProbs;
+
+ public TestWorkloadConf(TestOperation[] ops, float[] opProbs) {
+ this.ops = ops;
+ this.opProbs = opProbs;
+ }
+
+ public String toString() {
+ StringBuilder strBuilder = new StringBuilder();
+ for (TestOperation op : ops) {
+ strBuilder.append(op.toString());
+ strBuilder.append(',');
+ }
+ strBuilder.deleteCharAt(strBuilder.length() - 1);
+ return strBuilder.toString();
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexMultiThreadTestDriver.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexMultiThreadTestDriver.java
new file mode 100644
index 0000000..8c1d06f
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexMultiThreadTestDriver.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
+
+@SuppressWarnings("rawtypes")
+public class TreeIndexMultiThreadTestDriver {
+ private static final int RANDOM_SEED = 50;
+ // Means no additional payload. Only the specified fields.
+ private static final int PAYLOAD_SIZE = 0;
+ private final TestOperationSelector opSelector;
+ private final ISerializerDeserializer[] fieldSerdes;
+ private final ITreeIndex index;
+ private final ITreeIndexTestWorkerFactory workerFactory;
+
+ public TreeIndexMultiThreadTestDriver(ITreeIndex index, ITreeIndexTestWorkerFactory workerFactory,
+ ISerializerDeserializer[] fieldSerdes, TestOperation[] ops, float[] opProbs) {
+ this.index = index;
+ this.workerFactory = workerFactory;
+ this.fieldSerdes = fieldSerdes;
+ this.opSelector = new TestOperationSelector(ops, opProbs);
+ }
+
+ public void init(int fileId) throws HyracksDataException {
+ index.create(fileId);
+ index.open(fileId);
+ }
+
+ public long[] run(int numThreads, int numRepeats, int numOps, int batchSize) throws InterruptedException, TreeIndexException {
+ int numBatches = numOps / batchSize;
+ int threadNumBatches = numBatches / numThreads;
+ if (threadNumBatches <= 0) {
+ throw new TreeIndexException("Inconsistent parameters given. Need at least one batch per thread.");
+ }
+ long[] times = new long[numRepeats];
+ for (int i = 0; i < numRepeats; i++) {
+ DataGenThread dataGen = createDatagenThread(numThreads, numBatches, batchSize);
+ dataGen.start();
+ // Wait until the tupleBatchQueue is filled to capacity.
+ while (dataGen.tupleBatchQueue.remainingCapacity() != 0 && dataGen.tupleBatchQueue.size() != numBatches) {
+ Thread.sleep(10);
+ }
+
+ // Start worker threads.
+ AbstractTreeIndexTestWorker[] workers = new AbstractTreeIndexTestWorker[numThreads];
+ long start = System.currentTimeMillis();
+ for (int j = 0; j < numThreads; j++) {
+ workers[j] = workerFactory.create(dataGen, opSelector, index, threadNumBatches);
+ workers[j].start();
+ }
+ // Join worker threads.
+ for (int j = 0; j < numThreads; j++) {
+ workers[j].join();
+ }
+ long end = System.currentTimeMillis();
+ times[i] = end - start;
+ }
+ return times;
+ }
+
+ public void deinit() throws HyracksDataException {
+ index.close();
+ }
+
+ // To allow subclasses to override the data gen params.
+ public DataGenThread createDatagenThread(int numThreads, int numBatches, int batchSize) {
+ return new DataGenThread(numThreads, numBatches, batchSize, fieldSerdes, PAYLOAD_SIZE, RANDOM_SEED, 2*numThreads, false);
+ }
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexTestContext.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexTestContext.java
new file mode 100644
index 0000000..bc5312c
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexTestContext.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common;
+
+import java.util.Collection;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+
+@SuppressWarnings("rawtypes")
+public abstract class TreeIndexTestContext<T extends CheckTuple> implements ITreeIndexTestContext<T> {
+ protected final ISerializerDeserializer[] fieldSerdes;
+ protected final ITreeIndex treeIndex;
+ protected final ArrayTupleBuilder tupleBuilder;
+ protected final ArrayTupleReference tuple = new ArrayTupleReference();
+ protected final IIndexAccessor indexAccessor;
+
+ public TreeIndexTestContext(ISerializerDeserializer[] fieldSerdes, ITreeIndex treeIndex) {
+ this.fieldSerdes = fieldSerdes;
+ this.treeIndex = treeIndex;
+ this.indexAccessor = (IIndexAccessor) treeIndex.createAccessor();
+ this.tupleBuilder = new ArrayTupleBuilder(fieldSerdes.length);
+ }
+
+ @Override
+ public int getFieldCount() {
+ return fieldSerdes.length;
+ }
+
+ @Override
+ public IIndexAccessor getIndexAccessor() {
+ return indexAccessor;
+ }
+
+ @Override
+ public ArrayTupleReference getTuple() {
+ return tuple;
+ }
+
+ @Override
+ public ArrayTupleBuilder getTupleBuilder() {
+ return tupleBuilder;
+ }
+
+ @Override
+ public ISerializerDeserializer[] getFieldSerdes() {
+ return fieldSerdes;
+ }
+
+ @Override
+ public ITreeIndex getIndex() {
+ return treeIndex;
+ }
+
+ @Override
+ public void insertCheckTuple(T checkTuple, Collection<T> checkTuples) {
+ checkTuples.add(checkTuple);
+ }
+
+ @Override
+ public void deleteCheckTuple(T checkTuple, Collection<T> checkTuples) {
+ checkTuples.remove(checkTuple);
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexTestUtils.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexTestUtils.java
new file mode 100644
index 0000000..d16553a
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/common/TreeIndexTestUtils.java
@@ -0,0 +1,283 @@
+package edu.uci.ics.hyracks.storage.am.common;
+
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Random;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+
+@SuppressWarnings("rawtypes")
+public abstract class TreeIndexTestUtils {
+ private static final Logger LOGGER = Logger.getLogger(TreeIndexTestUtils.class.getName());
+
+ protected abstract CheckTuple createCheckTuple(int numFields, int numKeyFields);
+
+ protected abstract ISearchPredicate createNullSearchPredicate();
+
+ public abstract void checkExpectedResults(ITreeIndexCursor cursor, Collection checkTuples,
+ ISerializerDeserializer[] fieldSerdes, int keyFieldCount, Iterator<CheckTuple> checkIter) throws Exception;
+
+ protected abstract CheckTuple createIntCheckTuple(int[] fieldValues, int numKeyFields);
+
+ protected abstract void setIntKeyFields(int[] fieldValues, int numKeyFields, int maxValue, Random rnd);
+
+ protected abstract void setIntPayloadFields(int[] fieldValues, int numKeyFields, int numFields);
+
+ protected abstract Collection createCheckTuplesCollection();
+
+ protected abstract ArrayTupleBuilder createDeleteTupleBuilder(ITreeIndexTestContext ctx);
+
+ // See if tuple with corresponding checkTuple exists in ctx.checkTuples.
+ protected abstract boolean checkDiskOrderScanResult(ITupleReference tuple, CheckTuple checkTuple,
+ ITreeIndexTestContext ctx) throws HyracksDataException;
+
+ @SuppressWarnings("unchecked")
+ public static void createTupleFromCheckTuple(CheckTuple checkTuple, ArrayTupleBuilder tupleBuilder,
+ ArrayTupleReference tuple, ISerializerDeserializer[] fieldSerdes) throws HyracksDataException {
+ int fieldCount = tupleBuilder.getFieldEndOffsets().length;
+ DataOutput dos = tupleBuilder.getDataOutput();
+ tupleBuilder.reset();
+ for (int i = 0; i < fieldCount; i++) {
+ fieldSerdes[i].serialize(checkTuple.get(i), dos);
+ tupleBuilder.addFieldEndOffset();
+ }
+ tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
+ }
+
+ @SuppressWarnings("unchecked")
+ public CheckTuple createCheckTupleFromTuple(ITupleReference tuple, ISerializerDeserializer[] fieldSerdes,
+ int numKeys) throws HyracksDataException {
+ CheckTuple checkTuple = createCheckTuple(fieldSerdes.length, numKeys);
+ int fieldCount = Math.min(fieldSerdes.length, tuple.getFieldCount());
+ for (int i = 0; i < fieldCount; i++) {
+ ByteArrayInputStream inStream = new ByteArrayInputStream(tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i));
+ DataInput dataIn = new DataInputStream(inStream);
+ Comparable fieldObj = (Comparable) fieldSerdes[i].deserialize(dataIn);
+ checkTuple.add(fieldObj);
+ }
+ return checkTuple;
+ }
+
+ @SuppressWarnings("unchecked")
+ public void checkScan(ITreeIndexTestContext ctx) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Testing Scan.");
+ }
+ ITreeIndexCursor scanCursor = (ITreeIndexCursor) ctx.getIndexAccessor().createSearchCursor();
+ ISearchPredicate nullPred = createNullSearchPredicate();
+ ctx.getIndexAccessor().search(scanCursor, nullPred);
+ Iterator<CheckTuple> checkIter = ctx.getCheckTuples().iterator();
+ checkExpectedResults(scanCursor, ctx.getCheckTuples(), ctx.getFieldSerdes(), ctx.getKeyFieldCount(), checkIter);
+ }
+
+ public void checkDiskOrderScan(ITreeIndexTestContext ctx) throws Exception {
+ try {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Testing Disk-Order Scan.");
+ }
+ ITreeIndexAccessor treeIndexAccessor = (ITreeIndexAccessor) ctx.getIndexAccessor();
+ ITreeIndexCursor diskOrderCursor = treeIndexAccessor.createDiskOrderScanCursor();
+ treeIndexAccessor.diskOrderScan(diskOrderCursor);
+ int actualCount = 0;
+ try {
+ while (diskOrderCursor.hasNext()) {
+ diskOrderCursor.next();
+ ITupleReference tuple = diskOrderCursor.getTuple();
+ CheckTuple checkTuple = createCheckTupleFromTuple(tuple, ctx.getFieldSerdes(),
+ ctx.getKeyFieldCount());
+ if (!checkDiskOrderScanResult(tuple, checkTuple, ctx)) {
+ fail("Disk-order scan returned unexpected answer: " + checkTuple.toString());
+ }
+ actualCount++;
+ }
+ if (actualCount < ctx.getCheckTuples().size()) {
+ fail("Disk-order scan returned fewer answers than expected.\nExpected: "
+ + ctx.getCheckTuples().size() + "\nActual : " + actualCount);
+ }
+ if (actualCount > ctx.getCheckTuples().size()) {
+ fail("Disk-order scan returned more answers than expected.\nExpected: "
+ + ctx.getCheckTuples().size() + "\nActual : " + actualCount);
+ }
+ } finally {
+ diskOrderCursor.close();
+ }
+ } catch (UnsupportedOperationException e) {
+ // Ignore exception because some indexes, e.g. the LSMTrees, don't
+ // support disk-order scan.
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Ignoring disk-order scan since it's not supported.");
+ }
+ } catch (ClassCastException e) {
+ // Ignore exception because IIndexAccessor sometimes isn't
+ // an ITreeIndexAccessor, e.g., for the LSMBTree.
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Ignoring disk-order scan since it's not supported.");
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public void insertIntTuples(ITreeIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = ctx.getFieldCount();
+ int numKeyFields = ctx.getKeyFieldCount();
+ int[] fieldValues = new int[ctx.getFieldCount()];
+ // Scale range of values according to number of keys.
+ // For example, for 2 keys we want the square root of numTuples, for 3
+ // keys the cube root of numTuples, etc.
+ int maxValue = (int) Math.ceil(Math.pow(numTuples, 1.0 / (double) numKeyFields));
+ for (int i = 0; i < numTuples; i++) {
+ // Set keys.
+ setIntKeyFields(fieldValues, numKeyFields, maxValue, rnd);
+ // Set values.
+ setIntPayloadFields(fieldValues, numKeyFields, fieldCount);
+ TupleUtils.createIntegerTuple(ctx.getTupleBuilder(), ctx.getTuple(), fieldValues);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
+ LOGGER.info("Inserting Tuple " + (i + 1) + "/" + numTuples);
+ }
+ }
+ try {
+ ctx.getIndexAccessor().insert(ctx.getTuple());
+ ctx.insertCheckTuple(createIntCheckTuple(fieldValues, ctx.getKeyFieldCount()), ctx.getCheckTuples());
+ } catch (TreeIndexException e) {
+ // We set expected values only after insertion succeeds because
+ // we ignore duplicate keys.
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public void upsertIntTuples(ITreeIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = ctx.getFieldCount();
+ int numKeyFields = ctx.getKeyFieldCount();
+ int[] fieldValues = new int[ctx.getFieldCount()];
+ // Scale range of values according to number of keys.
+ // For example, for 2 keys we want the square root of numTuples, for 3
+ // keys the cube root of numTuples, etc.
+ int maxValue = (int) Math.ceil(Math.pow(numTuples, 1.0 / (double) numKeyFields));
+ for (int i = 0; i < numTuples; i++) {
+ // Set keys.
+ setIntKeyFields(fieldValues, numKeyFields, maxValue, rnd);
+ // Set values.
+ setIntPayloadFields(fieldValues, numKeyFields, fieldCount);
+ TupleUtils.createIntegerTuple(ctx.getTupleBuilder(), ctx.getTuple(), fieldValues);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
+ LOGGER.info("Inserting Tuple " + (i + 1) + "/" + numTuples);
+ }
+ }
+ try {
+ ctx.getIndexAccessor().upsert(ctx.getTuple());
+ ctx.insertCheckTuple(createIntCheckTuple(fieldValues, ctx.getKeyFieldCount()), ctx.getCheckTuples());
+ } catch (TreeIndexException e) {
+ // We set expected values only after insertion succeeds because
+ // we ignore duplicate keys.
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public void bulkLoadIntTuples(ITreeIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = ctx.getFieldCount();
+ int numKeyFields = ctx.getKeyFieldCount();
+ int[] fieldValues = new int[ctx.getFieldCount()];
+ int maxValue = (int) Math.ceil(Math.pow(numTuples, 1.0 / (double) numKeyFields));
+ Collection<CheckTuple> tmpCheckTuples = createCheckTuplesCollection();
+ for (int i = 0; i < numTuples; i++) {
+ // Set keys.
+ setIntKeyFields(fieldValues, numKeyFields, maxValue, rnd);
+ // Set values.
+ setIntPayloadFields(fieldValues, numKeyFields, fieldCount);
+
+ // Set expected values. (We also use these as the pre-sorted stream
+ // for ordered indexes bulk loading).
+ ctx.insertCheckTuple(createIntCheckTuple(fieldValues, ctx.getKeyFieldCount()), tmpCheckTuples);
+ }
+ bulkLoadCheckTuples(ctx, tmpCheckTuples);
+
+ // Add tmpCheckTuples to ctx check tuples for comparing searches.
+ for (CheckTuple checkTuple : tmpCheckTuples) {
+ ctx.insertCheckTuple(checkTuple, ctx.getCheckTuples());
+ }
+ }
+
+ public static void bulkLoadCheckTuples(ITreeIndexTestContext ctx, Collection<CheckTuple> checkTuples)
+ throws HyracksDataException, IndexException {
+ int fieldCount = ctx.getFieldCount();
+ int numTuples = checkTuples.size();
+ ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ // Perform bulk load.
+ IIndexBulkLoadContext bulkLoadCtx = ctx.getIndex().beginBulkLoad(0.7f);
+ int c = 1;
+ for (CheckTuple checkTuple : checkTuples) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (c % (numTuples / 10) == 0) {
+ LOGGER.info("Bulk Loading Tuple " + c + "/" + numTuples);
+ }
+ }
+ createTupleFromCheckTuple(checkTuple, tupleBuilder, tuple, ctx.getFieldSerdes());
+ ctx.getIndex().bulkLoadAddTuple(tuple, bulkLoadCtx);
+ c++;
+ }
+ ctx.getIndex().endBulkLoad(bulkLoadCtx);
+ }
+
+ @SuppressWarnings("unchecked")
+ public void deleteTuples(ITreeIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
+ ArrayTupleBuilder deleteTupleBuilder = createDeleteTupleBuilder(ctx);
+ ArrayTupleReference deleteTuple = new ArrayTupleReference();
+ int numCheckTuples = ctx.getCheckTuples().size();
+ // Copy CheckTuple references into array, so we can randomly pick from
+ // there.
+ CheckTuple[] checkTuples = new CheckTuple[numCheckTuples];
+ int idx = 0;
+ Iterator<CheckTuple> iter = ctx.getCheckTuples().iterator();
+ while (iter.hasNext()) {
+ CheckTuple checkTuple = iter.next();
+ checkTuples[idx++] = checkTuple;
+ }
+
+ for (int i = 0; i < numTuples && numCheckTuples > 0; i++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
+ LOGGER.info("Deleting Tuple " + (i + 1) + "/" + numTuples);
+ }
+ }
+ int checkTupleIdx = Math.abs(rnd.nextInt() % numCheckTuples);
+ CheckTuple checkTuple = checkTuples[checkTupleIdx];
+ createTupleFromCheckTuple(checkTuple, deleteTupleBuilder, deleteTuple, ctx.getFieldSerdes());
+ ctx.getIndexAccessor().delete(deleteTuple);
+
+ // Remove check tuple from expected results.
+ ctx.deleteCheckTuple(checkTuple, ctx.getCheckTuples());
+
+ // Swap with last "valid" CheckTuple.
+ CheckTuple tmp = checkTuples[numCheckTuples - 1];
+ checkTuples[numCheckTuples - 1] = checkTuple;
+ checkTuples[checkTupleIdx] = tmp;
+ numCheckTuples--;
+ }
+ }
+
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeBulkLoadTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeBulkLoadTest.java
new file mode 100644
index 0000000..198ac58
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeBulkLoadTest.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+
+@SuppressWarnings("rawtypes")
+public abstract class AbstractRTreeBulkLoadTest extends AbstractRTreeTestDriver {
+
+ private final RTreeTestUtils rTreeTestUtils;
+ private final int bulkLoadRounds;
+
+ public AbstractRTreeBulkLoadTest(int bulkLoadRounds) {
+ this.bulkLoadRounds = bulkLoadRounds;
+ this.rTreeTestUtils = new RTreeTestUtils();
+ }
+
+ @Override
+ protected void runTest(ISerializerDeserializer[] fieldSerdes,
+ IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, ITupleReference key) throws Exception {
+ AbstractRTreeTestContext ctx = createTestContext(fieldSerdes, valueProviderFactories, numKeys);
+ for (int i = 0; i < bulkLoadRounds; i++) {
+ // We assume all fieldSerdes are of the same type. Check the first
+ // one to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ rTreeTestUtils.bulkLoadIntTuples(ctx, numTuplesToInsert, getRandom());
+ } else if (fieldSerdes[0] instanceof DoubleSerializerDeserializer) {
+ rTreeTestUtils.bulkLoadDoubleTuples(ctx, numTuplesToInsert, getRandom());
+ }
+
+ rTreeTestUtils.checkScan(ctx);
+ rTreeTestUtils.checkDiskOrderScan(ctx);
+ rTreeTestUtils.checkRangeSearch(ctx, key);
+ }
+ ctx.getIndex().close();
+ }
+
+ @Override
+ protected String getTestOpName() {
+ return "BulkLoad";
+ }
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeDeleteTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeDeleteTest.java
new file mode 100644
index 0000000..e70f433
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeDeleteTest.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+
+@SuppressWarnings("rawtypes")
+public abstract class AbstractRTreeDeleteTest extends AbstractRTreeTestDriver {
+
+ private final RTreeTestUtils rTreeTestUtils;
+
+ private static final int numInsertRounds = 2;
+ private static final int numDeleteRounds = 2;
+
+ public AbstractRTreeDeleteTest() {
+ this.rTreeTestUtils = new RTreeTestUtils();
+ }
+
+ @Override
+ protected void runTest(ISerializerDeserializer[] fieldSerdes,
+ IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, ITupleReference key) throws Exception {
+ AbstractRTreeTestContext ctx = createTestContext(fieldSerdes, valueProviderFactories, numKeys);
+ for (int i = 0; i < numInsertRounds; i++) {
+ // We assume all fieldSerdes are of the same type. Check the first
+ // one to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ rTreeTestUtils.insertIntTuples(ctx, numTuplesToInsert, getRandom());
+ } else if (fieldSerdes[0] instanceof DoubleSerializerDeserializer) {
+ rTreeTestUtils.insertDoubleTuples(ctx, numTuplesToInsert, getRandom());
+ }
+ int numTuplesPerDeleteRound = (int) Math
+ .ceil((float) ctx.getCheckTuples().size() / (float) numDeleteRounds);
+ for (int j = 0; j < numDeleteRounds; j++) {
+ rTreeTestUtils.deleteTuples(ctx, numTuplesPerDeleteRound, getRandom());
+ rTreeTestUtils.checkScan(ctx);
+ rTreeTestUtils.checkDiskOrderScan(ctx);
+ rTreeTestUtils.checkRangeSearch(ctx, key);
+ }
+ }
+ ctx.getIndex().close();
+ }
+
+ @Override
+ protected String getTestOpName() {
+ return "Delete";
+ }
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
new file mode 100644
index 0000000..7192c53
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
@@ -0,0 +1,575 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree;
+
+import java.util.Random;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.SearchPredicate;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
+
+@SuppressWarnings("rawtypes")
+public abstract class AbstractRTreeExamplesTest {
+ protected static final Logger LOGGER = Logger.getLogger(AbstractRTreeExamplesTest.class.getName());
+ protected final Random rnd = new Random(50);
+
+ protected abstract ITreeIndex createTreeIndex(ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
+ IPrimitiveValueProviderFactory[] valueProviderFactories) throws TreeIndexException;
+
+ protected abstract int getIndexFileId();
+
+ /**
+ * Two Dimensions Example.
+ *
+ * Create an RTree index of two dimensions, where they keys are of type
+ * integer, and the payload is two integer values. Fill index with random
+ * values using insertions (not bulk load). Perform scans and range search.
+ */
+ @Test
+ public void twoDimensionsExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Fixed-Length Key,Value Example.");
+ }
+
+ // Declare fields.
+ int fieldCount = 6;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[3] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[4] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[5] = IntegerPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE };
+
+ // Declare RTree keys.
+ int rtreeKeyFieldCount = 4;
+ IBinaryComparatorFactory[] rtreeCmpFactories = new IBinaryComparatorFactory[rtreeKeyFieldCount];
+ rtreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ rtreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ rtreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ // Declare BTree keys, this will only be used for LSMRTree
+ int btreeKeyFieldCount = 6;
+ IBinaryComparatorFactory[] btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[5] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ rtreeCmpFactories.length, IntegerPointable.FACTORY);
+
+ int indexFileId = getIndexFileId();
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories);
+ treeIndex.create(indexFileId);
+ treeIndex.open(indexFileId);
+
+ long start = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inserting into tree...");
+ }
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor();
+ int numInserts = 10000;
+ for (int i = 0; i < numInserts; i++) {
+ int p1x = rnd.nextInt();
+ int p1y = rnd.nextInt();
+ int p2x = rnd.nextInt();
+ int p2y = rnd.nextInt();
+
+ int pk1 = 5;
+ int pk2 = 10;
+
+ TupleUtils.createIntegerTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x),
+ Math.max(p1y, p2y), pk1, pk2);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y) + ", " + pk1 + ", " + pk2);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
+ }
+
+ scan(indexAccessor, fieldSerdes);
+ diskOrderScan(indexAccessor, fieldSerdes);
+
+ // Build key.
+ ArrayTupleBuilder keyTb = new ArrayTupleBuilder(rtreeKeyFieldCount);
+ ArrayTupleReference key = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(keyTb, key, -1000, -1000, 1000, 1000);
+
+ rangeSearch(rtreeCmpFactories, indexAccessor, fieldSerdes, key);
+
+ treeIndex.close();
+ }
+
+ /**
+ * Two Dimensions Example.
+ *
+ * Create an RTree index of three dimensions, where they keys are of type
+ * double, and the payload is one double value. Fill index with random
+ * values using insertions (not bulk load). Perform scans and range search.
+ */
+ @Test
+ public void threeDimensionsExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Fixed-Length Key,Value Example.");
+ }
+
+ // Declare fields.
+ int fieldCount = 7;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = DoublePointable.TYPE_TRAITS;
+ typeTraits[1] = DoublePointable.TYPE_TRAITS;
+ typeTraits[2] = DoublePointable.TYPE_TRAITS;
+ typeTraits[3] = DoublePointable.TYPE_TRAITS;
+ typeTraits[4] = DoublePointable.TYPE_TRAITS;
+ typeTraits[5] = DoublePointable.TYPE_TRAITS;
+ typeTraits[6] = DoublePointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
+
+ // Declare RTree keys.
+ int rtreeKeyFieldCount = 6;
+ IBinaryComparatorFactory[] rtreeCmpFactories = new IBinaryComparatorFactory[rtreeKeyFieldCount];
+ rtreeCmpFactories[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ rtreeCmpFactories[1] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ rtreeCmpFactories[2] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ rtreeCmpFactories[4] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ rtreeCmpFactories[5] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+
+ // Declare RTree keys.
+ int btreeKeyFieldCount = 7;
+ IBinaryComparatorFactory[] btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeCmpFactories[5] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeCmpFactories[6] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ rtreeCmpFactories.length, DoublePointable.FACTORY);
+
+ int indexFileId = getIndexFileId();
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories);
+ treeIndex.create(indexFileId);
+ treeIndex.open(indexFileId);
+
+ long start = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inserting into tree...");
+ }
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor();
+ int numInserts = 10000;
+ for (int i = 0; i < numInserts; i++) {
+ double p1x = rnd.nextDouble();
+ double p1y = rnd.nextDouble();
+ double p1z = rnd.nextDouble();
+ double p2x = rnd.nextDouble();
+ double p2y = rnd.nextDouble();
+ double p2z = rnd.nextDouble();
+
+ double pk = 5.0;
+
+ TupleUtils.createDoubleTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.min(p1z, p2z),
+ Math.max(p1x, p2x), Math.max(p1y, p2y), Math.max(p1z, p2z), pk);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.min(p1z, p2z) + " " + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y) + " "
+ + Math.max(p1z, p2z) + ", " + pk);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
+ }
+
+ scan(indexAccessor, fieldSerdes);
+ diskOrderScan(indexAccessor, fieldSerdes);
+
+ // Build key.
+ ArrayTupleBuilder keyTb = new ArrayTupleBuilder(rtreeKeyFieldCount);
+ ArrayTupleReference key = new ArrayTupleReference();
+ TupleUtils.createDoubleTuple(keyTb, key, -1000.0, -1000.0, -1000.0, 1000.0, 1000.0, 1000.0);
+
+ rangeSearch(rtreeCmpFactories, indexAccessor, fieldSerdes, key);
+
+ treeIndex.close();
+ }
+
+ /**
+ * Deletion Example.
+ *
+ * Create an RTree index of two dimensions, where they keys are of type
+ * integer, and the payload is one integer value. Fill index with random
+ * values using insertions, then delete entries one-by-one. Repeat procedure
+ * a few times on same RTree.
+ */
+ @Test
+ public void deleteExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Deletion Example");
+ }
+
+ // Declare fields.
+ int fieldCount = 5;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[3] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[4] = IntegerPointable.TYPE_TRAITS;
+
+ // Declare RTree keys.
+ int rtreeKeyFieldCount = 4;
+ IBinaryComparatorFactory[] rtreeCmpFactories = new IBinaryComparatorFactory[rtreeKeyFieldCount];
+ rtreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ rtreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ rtreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ // Declare BTree keys.
+ int btreeKeyFieldCount = 5;
+ IBinaryComparatorFactory[] btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ rtreeCmpFactories.length, IntegerPointable.FACTORY);
+
+ int indexFileId = getIndexFileId();
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories);
+ treeIndex.create(indexFileId);
+ treeIndex.open(indexFileId);
+
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor();
+
+ int runs = 3;
+ for (int run = 0; run < runs; run++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Deletion example run: " + (run + 1) + "/" + runs);
+ LOGGER.info("Inserting into tree...");
+ }
+
+ int numInserts = 10000;
+ int[] p1xs = new int[numInserts];
+ int[] p1ys = new int[numInserts];
+ int[] p2xs = new int[numInserts];
+ int[] p2ys = new int[numInserts];
+ int[] pks = new int[numInserts];
+ int insDone = 0;
+
+ int[] insDoneCmp = new int[numInserts];
+ for (int i = 0; i < numInserts; i++) {
+ int p1x = rnd.nextInt();
+ int p1y = rnd.nextInt();
+ int p2x = rnd.nextInt();
+ int p2y = rnd.nextInt();
+ int pk = 5;
+
+ p1xs[i] = Math.min(p1x, p2x);
+ p1ys[i] = Math.min(p1y, p2y);
+ p2xs[i] = Math.max(p1x, p2x);
+ p2ys[i] = Math.max(p1y, p2y);
+ pks[i] = pk;
+
+ TupleUtils.createIntegerTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x),
+ Math.max(p1y, p2y), pk);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + i);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ insDoneCmp[i] = insDone;
+ }
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Deleting from tree...");
+ }
+ int delDone = 0;
+ for (int i = 0; i < numInserts; i++) {
+ TupleUtils.createIntegerTuple(tb, tuple, p1xs[i], p1ys[i], p2xs[i], p2ys[i], pks[i]);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Deleting " + i);
+ }
+ }
+ try {
+ indexAccessor.delete(tuple);
+ delDone++;
+ } catch (TreeIndexException e) {
+ }
+ if (insDoneCmp[i] != delDone) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("INCONSISTENT STATE, ERROR IN DELETION EXAMPLE.");
+ LOGGER.info("INSDONECMP: " + insDoneCmp[i] + " " + delDone);
+ }
+ break;
+ }
+ }
+ if (insDone != delDone) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("ERROR! INSDONE: " + insDone + " DELDONE: " + delDone);
+ }
+ break;
+ }
+ }
+ treeIndex.close();
+ }
+
+ /**
+ * Bulk load example.
+ *
+ * Load a tree with 10,000 tuples.
+ *
+ */
+ @Test
+ public void bulkLoadExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Bulk load example");
+ }
+ // Declare fields.
+ int fieldCount = 5;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[3] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[4] = IntegerPointable.TYPE_TRAITS;
+
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+
+ // Declare RTree keys.
+ int rtreeKeyFieldCount = 4;
+ IBinaryComparatorFactory[] rtreeCmpFactories = new IBinaryComparatorFactory[rtreeKeyFieldCount];
+ rtreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ rtreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ rtreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ // Declare BTree keys.
+ int btreeKeyFieldCount = 5;
+ IBinaryComparatorFactory[] btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ rtreeCmpFactories.length, IntegerPointable.FACTORY);
+
+ int indexFileId = getIndexFileId();
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories);
+ treeIndex.create(indexFileId);
+ treeIndex.open(indexFileId);
+
+ // Load records.
+ int numInserts = 10000;
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Bulk loading " + numInserts + " tuples");
+ }
+ long start = System.currentTimeMillis();
+ IIndexBulkLoadContext bulkLoadCtx = treeIndex.beginBulkLoad(0.7f);
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+
+ for (int i = 0; i < numInserts; i++) {
+ int p1x = rnd.nextInt();
+ int p1y = rnd.nextInt();
+ int p2x = rnd.nextInt();
+ int p2y = rnd.nextInt();
+
+ int pk = 5;
+
+ TupleUtils.createIntegerTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x),
+ Math.max(p1y, p2y), pk);
+ treeIndex.bulkLoadAddTuple(tuple, bulkLoadCtx);
+ }
+
+ treeIndex.endBulkLoad(bulkLoadCtx);
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(numInserts + " tuples loaded in " + (end - start) + "ms");
+ }
+
+ IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor();
+
+ // Build key.
+ ArrayTupleBuilder keyTb = new ArrayTupleBuilder(rtreeKeyFieldCount);
+ ArrayTupleReference key = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(keyTb, key, -1000, -1000, 1000, 1000);
+
+ rangeSearch(rtreeCmpFactories, indexAccessor, fieldSerdes, key);
+
+ treeIndex.close();
+ }
+
+ private void scan(IIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Scan:");
+ }
+ ITreeIndexCursor scanCursor = (ITreeIndexCursor) indexAccessor.createSearchCursor();
+ SearchPredicate nullPred = new SearchPredicate(null, null);
+ indexAccessor.search(scanCursor, nullPred);
+ try {
+ while (scanCursor.hasNext()) {
+ scanCursor.next();
+ ITupleReference frameTuple = scanCursor.getTuple();
+ String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rec);
+ }
+ }
+ } finally {
+ scanCursor.close();
+ }
+ }
+
+ private void diskOrderScan(IIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes)
+ throws Exception {
+ try {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Disk-Order Scan:");
+ }
+ ITreeIndexAccessor treeIndexAccessor = (ITreeIndexAccessor) indexAccessor;
+ TreeDiskOrderScanCursor diskOrderCursor = (TreeDiskOrderScanCursor) treeIndexAccessor
+ .createDiskOrderScanCursor();
+ treeIndexAccessor.diskOrderScan(diskOrderCursor);
+ try {
+ while (diskOrderCursor.hasNext()) {
+ diskOrderCursor.next();
+ ITupleReference frameTuple = diskOrderCursor.getTuple();
+ String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rec);
+ }
+ }
+ } finally {
+ diskOrderCursor.close();
+ }
+ } catch (UnsupportedOperationException e) {
+ // Ignore exception because some indexes, e.g. the LSMRTree, don't
+ // support disk-order scan.
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Ignoring disk-order scan since it's not supported.");
+ }
+ } catch (ClassCastException e) {
+ // Ignore exception because IIndexAccessor sometimes isn't
+ // an ITreeIndexAccessor, e.g., for the LSMRTree.
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Ignoring disk-order scan since it's not supported.");
+ }
+ }
+ }
+
+ private void rangeSearch(IBinaryComparatorFactory[] cmpFactories, IIndexAccessor indexAccessor,
+ ISerializerDeserializer[] fieldSerdes, ITupleReference key) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ String kString = TupleUtils.printTuple(key, fieldSerdes);
+ LOGGER.info("Range-Search using key: " + kString);
+ }
+ ITreeIndexCursor rangeCursor = (ITreeIndexCursor) indexAccessor.createSearchCursor();
+ MultiComparator cmp = RTreeUtils.getSearchMultiComparator(cmpFactories, key);
+ SearchPredicate rangePred = new SearchPredicate(key, cmp);
+ indexAccessor.search(rangeCursor, rangePred);
+ try {
+ while (rangeCursor.hasNext()) {
+ rangeCursor.next();
+ ITupleReference frameTuple = rangeCursor.getTuple();
+ String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rec);
+ }
+ }
+ } finally {
+ rangeCursor.close();
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeInsertTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeInsertTest.java
new file mode 100644
index 0000000..cdd6ee0
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeInsertTest.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+
+/**
+ * Tests the RTree insert operation with integer and double fields using various
+ * numbers of dimensions and payload fields.
+ *
+ * Each tests first fills an RTree with randomly generated tuples. We compare
+ * the following operations against expected results: 1. RTree scan. 3.
+ * Disk-order scan. 4. Range search.
+ *
+ */
+@SuppressWarnings("rawtypes")
+public abstract class AbstractRTreeInsertTest extends AbstractRTreeTestDriver {
+
+ private final RTreeTestUtils rTreeTestUtils;
+
+ public AbstractRTreeInsertTest() {
+ this.rTreeTestUtils = new RTreeTestUtils();
+ }
+
+ @Override
+ protected void runTest(ISerializerDeserializer[] fieldSerdes,
+ IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, ITupleReference key) throws Exception {
+ AbstractRTreeTestContext ctx = createTestContext(fieldSerdes, valueProviderFactories, numKeys);
+ // We assume all fieldSerdes are of the same type. Check the first one
+ // to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ rTreeTestUtils.insertIntTuples(ctx, numTuplesToInsert, getRandom());
+ } else if (fieldSerdes[0] instanceof DoubleSerializerDeserializer) {
+ rTreeTestUtils.insertDoubleTuples(ctx, numTuplesToInsert, getRandom());
+ }
+
+ rTreeTestUtils.checkScan(ctx);
+ rTreeTestUtils.checkDiskOrderScan(ctx);
+ rTreeTestUtils.checkRangeSearch(ctx, key);
+ ctx.getIndex().close();
+ }
+
+ @Override
+ protected String getTestOpName() {
+ return "Insert";
+ }
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeMultiThreadTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeMultiThreadTest.java
new file mode 100644
index 0000000..2c185a5
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeMultiThreadTest.java
@@ -0,0 +1,152 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree;
+
+import java.util.ArrayList;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
+import edu.uci.ics.hyracks.storage.am.common.ITreeIndexTestWorkerFactory;
+import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
+import edu.uci.ics.hyracks.storage.am.common.TestWorkloadConf;
+import edu.uci.ics.hyracks.storage.am.common.TreeIndexMultiThreadTestDriver;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
+
+@SuppressWarnings("rawtypes")
+public abstract class AbstractRTreeMultiThreadTest {
+
+ protected final Logger LOGGER = Logger.getLogger(AbstractRTreeMultiThreadTest.class.getName());
+
+ // Machine-specific number of threads to use for testing.
+ protected final int REGULAR_NUM_THREADS = Runtime.getRuntime().availableProcessors();
+ // Excessive number of threads for testing.
+ protected final int EXCESSIVE_NUM_THREADS = Runtime.getRuntime().availableProcessors() * 4;
+ protected final int NUM_OPERATIONS = 5000;
+
+ protected ArrayList<TestWorkloadConf> workloadConfs = getTestWorkloadConf();
+
+ protected abstract void setUp() throws HyracksException;
+
+ protected abstract void tearDown() throws HyracksDataException;
+
+ protected abstract ITreeIndex createTreeIndex(ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
+ IPrimitiveValueProviderFactory[] valueProviderFactories) throws TreeIndexException;
+
+ protected abstract int getFileId();
+
+ protected abstract ITreeIndexTestWorkerFactory getWorkerFactory();
+
+ protected abstract ArrayList<TestWorkloadConf> getTestWorkloadConf();
+
+ protected abstract String getIndexTypeName();
+
+ protected static float[] getUniformOpProbs(TestOperation[] ops) {
+ float[] opProbs = new float[ops.length];
+ for (int i = 0; i < ops.length; i++) {
+ opProbs[i] = 1.0f / (float) ops.length;
+ }
+ return opProbs;
+ }
+
+ protected void runTest(ISerializerDeserializer[] fieldSerdes,
+ IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, int numThreads,
+ TestWorkloadConf conf, String dataMsg) throws HyracksException, InterruptedException, TreeIndexException {
+ setUp();
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ String indexTypeName = getIndexTypeName();
+ LOGGER.info(indexTypeName + " MultiThread Test:\nData: " + dataMsg + "; Threads: " + numThreads
+ + "; Workload: " + conf.toString() + ".");
+ }
+
+ ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
+ IBinaryComparatorFactory[] rtreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeys);
+ IBinaryComparatorFactory[] btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes,
+ fieldSerdes.length);
+
+ ITreeIndex index = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories);
+ ITreeIndexTestWorkerFactory workerFactory = getWorkerFactory();
+
+ // 4 batches per thread.
+ int batchSize = (NUM_OPERATIONS / numThreads) / 4;
+
+ TreeIndexMultiThreadTestDriver driver = new TreeIndexMultiThreadTestDriver(index, workerFactory, fieldSerdes,
+ conf.ops, conf.opProbs);
+ driver.init(getFileId());
+ long[] times = driver.run(numThreads, 1, NUM_OPERATIONS, batchSize);
+ driver.deinit();
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("RTree MultiThread Test Time: " + times[0] + "ms");
+ }
+
+ tearDown();
+ }
+
+ @Test
+ public void twoDimensionsInt() throws InterruptedException, HyracksException, TreeIndexException {
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+
+ int numKeys = 4;
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ numKeys, IntegerPointable.FACTORY);
+
+ String dataMsg = "Two Dimensions Of Integer Values";
+
+ for (TestWorkloadConf conf : workloadConfs) {
+ runTest(fieldSerdes, valueProviderFactories, numKeys, REGULAR_NUM_THREADS, conf, dataMsg);
+ runTest(fieldSerdes, valueProviderFactories, numKeys, EXCESSIVE_NUM_THREADS, conf, dataMsg);
+ }
+ }
+
+ @Test
+ public void fourDimensionsDouble() throws InterruptedException, HyracksException, TreeIndexException {
+ ISerializerDeserializer[] fieldSerdes = { DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
+
+ int numKeys = 8;
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ numKeys, DoublePointable.FACTORY);
+
+ String dataMsg = "Four Dimensions Of Double Values";
+
+ for (TestWorkloadConf conf : workloadConfs) {
+ runTest(fieldSerdes, valueProviderFactories, numKeys, REGULAR_NUM_THREADS, conf, dataMsg);
+ runTest(fieldSerdes, valueProviderFactories, numKeys, EXCESSIVE_NUM_THREADS, conf, dataMsg);
+ }
+ }
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeTestContext.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeTestContext.java
new file mode 100644
index 0000000..9affc47
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeTestContext.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.common.TreeIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+
+@SuppressWarnings("rawtypes")
+public abstract class AbstractRTreeTestContext extends TreeIndexTestContext<RTreeCheckTuple> {
+ private final ArrayList<RTreeCheckTuple> checkTuples = new ArrayList<RTreeCheckTuple>();
+
+ public AbstractRTreeTestContext(ISerializerDeserializer[] fieldSerdes, ITreeIndex treeIndex) {
+ super(fieldSerdes, treeIndex);
+ }
+
+ @Override
+ public Collection<RTreeCheckTuple> getCheckTuples() {
+ return checkTuples;
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeTestDriver.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeTestDriver.java
new file mode 100644
index 0000000..10f4364
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeTestDriver.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree;
+
+import java.util.Random;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
+
+@SuppressWarnings("rawtypes")
+public abstract class AbstractRTreeTestDriver {
+ protected final Logger LOGGER = Logger.getLogger(AbstractRTreeTestDriver.class.getName());
+
+ protected static final int numTuplesToInsert = 5000;
+
+ protected abstract AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
+ IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys) throws Exception;
+
+ protected abstract Random getRandom();
+
+ protected abstract void runTest(ISerializerDeserializer[] fieldSerdes,
+ IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, ITupleReference key) throws Exception;
+
+ protected abstract String getTestOpName();
+
+ @Test
+ public void twoDimensionsInt() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("RTree " + getTestOpName() + " Test With Two Dimensions With Integer Keys.");
+ }
+
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+
+ int numKeys = 4;
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ numKeys, IntegerPointable.FACTORY);
+ // Range search, the rectangle bottom left coordinates are -1000, -1000
+ // and the top right coordinates are 1000, 1000
+ ITupleReference key = TupleUtils.createIntegerTuple(-1000, -1000, 1000, 1000);
+
+ runTest(fieldSerdes, valueProviderFactories, numKeys, key);
+
+ }
+
+ @Test
+ public void twoDimensionsDouble() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("RTree " + getTestOpName() + " Test With Two Dimensions With Double Keys.");
+ }
+
+ ISerializerDeserializer[] fieldSerdes = { DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
+
+ int numKeys = 4;
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ numKeys, DoublePointable.FACTORY);
+ // Range search, the rectangle bottom left coordinates are -1000.0,
+ // -1000.0 and the top right coordinates are 1000.0, 1000.0
+ ITupleReference key = TupleUtils.createDoubleTuple(-1000.0, -1000.0, 1000.0, 1000.0);
+
+ runTest(fieldSerdes, valueProviderFactories, numKeys, key);
+
+ }
+
+ @Test
+ public void fourDimensionsDouble() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("RTree " + getTestOpName() + " Test With Four Dimensions With Double Keys.");
+ }
+
+ ISerializerDeserializer[] fieldSerdes = { DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
+
+ int numKeys = 8;
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ numKeys, DoublePointable.FACTORY);
+ // Range search, the rectangle bottom left coordinates are -1000.0,
+ // -1000.0, -1000.0, -1000.0 and the top right coordinates are 1000.0,
+ // 1000.0, 1000.0, 1000.0
+ ITupleReference key = TupleUtils.createDoubleTuple(-1000.0, -1000.0, -1000.0, -1000.0, 1000.0, 1000.0, 1000.0,
+ 1000.0);
+
+ runTest(fieldSerdes, valueProviderFactories, numKeys, key);
+
+ }
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeCheckTuple.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeCheckTuple.java
new file mode 100644
index 0000000..98800e5
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeCheckTuple.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree;
+
+import edu.uci.ics.hyracks.storage.am.common.CheckTuple;
+
+@SuppressWarnings({ "rawtypes", "unchecked" })
+public class RTreeCheckTuple<T> extends CheckTuple {
+
+ public RTreeCheckTuple(int numFields, int numKeys) {
+ super(numFields, numKeys);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ RTreeCheckTuple<T> other = (RTreeCheckTuple<T>) o;
+ for (int i = 0; i < tuple.length; i++) {
+ int cmp = tuple[i].compareTo(other.get(i));
+ if (cmp != 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public boolean intersect(T o) {
+ RTreeCheckTuple<T> other = (RTreeCheckTuple<T>) o;
+ int maxFieldPos = numKeys / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ int cmp = tuple[i].compareTo(other.get(j));
+ if (cmp > 0) {
+ return false;
+ }
+ cmp = tuple[j].compareTo(other.get(i));
+ if (cmp < 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTestUtils.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTestUtils.java
new file mode 100644
index 0000000..a23f375
--- /dev/null
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTestUtils.java
@@ -0,0 +1,239 @@
+package edu.uci.ics.hyracks.storage.am.rtree;
+
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Random;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.common.CheckTuple;
+import edu.uci.ics.hyracks.storage.am.common.ITreeIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.common.TreeIndexTestUtils;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.SearchPredicate;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
+
+@SuppressWarnings("rawtypes")
+public class RTreeTestUtils extends TreeIndexTestUtils {
+ private static final Logger LOGGER = Logger.getLogger(RTreeTestUtils.class.getName());
+ private int intPayloadValue = 0;
+ private double doublePayloadValue = 0.0;
+
+ @SuppressWarnings("unchecked")
+ // Create a new ArrayList containing the elements satisfying the search key
+ public ArrayList<RTreeCheckTuple> getRangeSearchExpectedResults(ArrayList<RTreeCheckTuple> checkTuples,
+ RTreeCheckTuple key) {
+ ArrayList<RTreeCheckTuple> expectedResult = new ArrayList<RTreeCheckTuple>();
+ Iterator<RTreeCheckTuple> iter = checkTuples.iterator();
+ while (iter.hasNext()) {
+ RTreeCheckTuple t = iter.next();
+ if (t.intersect(key)) {
+ expectedResult.add(t);
+ }
+ }
+ return expectedResult;
+ }
+
+ public void checkRangeSearch(ITreeIndexTestContext ictx, ITupleReference key) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Testing Range Search.");
+ }
+ AbstractRTreeTestContext ctx = (AbstractRTreeTestContext) ictx;
+ MultiComparator cmp = RTreeUtils.getSearchMultiComparator(ctx.getComparatorFactories(), key);
+
+ ITreeIndexCursor searchCursor = (ITreeIndexCursor) ctx.getIndexAccessor().createSearchCursor();
+ SearchPredicate searchPred = new SearchPredicate(key, cmp);
+ ctx.getIndexAccessor().search(searchCursor, searchPred);
+
+ // Get the subset of elements from the expected set within given key
+ // range.
+ RTreeCheckTuple keyCheck = (RTreeCheckTuple) createCheckTupleFromTuple(key, ctx.getFieldSerdes(),
+ cmp.getKeyFieldCount());
+
+ ArrayList<RTreeCheckTuple> expectedResult = null;
+
+ expectedResult = getRangeSearchExpectedResults((ArrayList<RTreeCheckTuple>) ctx.getCheckTuples(), keyCheck);
+ checkExpectedResults(searchCursor, expectedResult, ctx.getFieldSerdes(), ctx.getKeyFieldCount(), null);
+ }
+
+ @SuppressWarnings("unchecked")
+ public void insertDoubleTuples(ITreeIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = ctx.getFieldCount();
+ int numKeyFields = ctx.getKeyFieldCount();
+ double[] fieldValues = new double[ctx.getFieldCount()];
+ // Scale range of values according to number of keys.
+ // For example, for 2 keys we want the square root of numTuples, for 3
+ // keys the cube root of numTuples, etc.
+ double maxValue = Math.ceil(Math.pow(numTuples, 1.0 / (double) numKeyFields));
+ for (int i = 0; i < numTuples; i++) {
+ // Set keys.
+ setDoubleKeyFields(fieldValues, numKeyFields, maxValue, rnd);
+ // Set values.
+ setDoublePayloadFields(fieldValues, numKeyFields, fieldCount);
+ TupleUtils.createDoubleTuple(ctx.getTupleBuilder(), ctx.getTuple(), fieldValues);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
+ LOGGER.info("Inserting Tuple " + (i + 1) + "/" + numTuples);
+ }
+ }
+ try {
+ ctx.getIndexAccessor().insert(ctx.getTuple());
+ ctx.insertCheckTuple(createDoubleCheckTuple(fieldValues, ctx.getKeyFieldCount()), ctx.getCheckTuples());
+ } catch (TreeIndexException e) {
+ // We set expected values only after insertion succeeds because
+ // we
+ // ignore duplicate keys.
+ }
+ }
+ }
+
+ private void setDoubleKeyFields(double[] fieldValues, int numKeyFields, double maxValue, Random rnd) {
+ int maxFieldPos = numKeyFields / 2;
+ for (int j = 0; j < maxFieldPos; j++) {
+ int k = maxFieldPos + j;
+ double firstValue = rnd.nextDouble() % maxValue;
+ double secondValue;
+ do {
+ secondValue = rnd.nextDouble() % maxValue;
+ } while (secondValue < firstValue);
+ fieldValues[j] = firstValue;
+ fieldValues[k] = secondValue;
+ }
+ }
+
+ private void setDoublePayloadFields(double[] fieldValues, int numKeyFields, int numFields) {
+ for (int j = numKeyFields; j < numFields; j++) {
+ fieldValues[j] = doublePayloadValue++;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ protected CheckTuple createDoubleCheckTuple(double[] fieldValues, int numKeyFields) {
+ RTreeCheckTuple<Double> checkTuple = new RTreeCheckTuple<Double>(fieldValues.length, numKeyFields);
+ for (double v : fieldValues) {
+ checkTuple.add(v);
+ }
+ return checkTuple;
+ }
+
+ @SuppressWarnings("unchecked")
+ public void bulkLoadDoubleTuples(ITreeIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = ctx.getFieldCount();
+ int numKeyFields = ctx.getKeyFieldCount();
+ double[] fieldValues = new double[ctx.getFieldCount()];
+ double maxValue = Math.ceil(Math.pow(numTuples, 1.0 / (double) numKeyFields));
+ Collection<CheckTuple> tmpCheckTuples = createCheckTuplesCollection();
+ for (int i = 0; i < numTuples; i++) {
+ // Set keys.
+ setDoubleKeyFields(fieldValues, numKeyFields, maxValue, rnd);
+ // Set values.
+ setDoublePayloadFields(fieldValues, numKeyFields, fieldCount);
+
+ // Set expected values.
+ ctx.insertCheckTuple(createDoubleCheckTuple(fieldValues, ctx.getKeyFieldCount()), tmpCheckTuples);
+ }
+ bulkLoadCheckTuples(ctx, tmpCheckTuples);
+
+ // Add tmpCheckTuples to ctx check tuples for comparing searches.
+ for (CheckTuple checkTuple : tmpCheckTuples) {
+ ctx.insertCheckTuple(checkTuple, ctx.getCheckTuples());
+ }
+ }
+
+ @Override
+ public void checkExpectedResults(ITreeIndexCursor cursor, Collection checkTuples,
+ ISerializerDeserializer[] fieldSerdes, int keyFieldCount, Iterator<CheckTuple> checkIter) throws Exception {
+ int actualCount = 0;
+ try {
+ while (cursor.hasNext()) {
+ cursor.next();
+ ITupleReference tuple = cursor.getTuple();
+ RTreeCheckTuple checkTuple = (RTreeCheckTuple) createCheckTupleFromTuple(tuple, fieldSerdes,
+ keyFieldCount);
+ if (!checkTuples.contains(checkTuple)) {
+ fail("Scan or range search returned unexpected answer: " + checkTuple.toString());
+ }
+ actualCount++;
+ }
+ if (actualCount < checkTuples.size()) {
+ fail("Scan or range search returned fewer answers than expected.\nExpected: " + checkTuples.size()
+ + "\nActual : " + actualCount);
+ }
+ if (actualCount > checkTuples.size()) {
+ fail("Scan or range search returned more answers than expected.\nExpected: " + checkTuples.size()
+ + "\nActual : " + actualCount);
+ }
+ } finally {
+ cursor.close();
+ }
+ }
+
+ @Override
+ protected CheckTuple createCheckTuple(int numFields, int numKeyFields) {
+ return new RTreeCheckTuple(numFields, numKeyFields);
+ }
+
+ @Override
+ protected ISearchPredicate createNullSearchPredicate() {
+ return new SearchPredicate(null, null);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected CheckTuple createIntCheckTuple(int[] fieldValues, int numKeyFields) {
+ RTreeCheckTuple<Integer> checkTuple = new RTreeCheckTuple<Integer>(fieldValues.length, numKeyFields);
+ for (int v : fieldValues) {
+ checkTuple.add(v);
+ }
+ return checkTuple;
+ }
+
+ @Override
+ protected void setIntKeyFields(int[] fieldValues, int numKeyFields, int maxValue, Random rnd) {
+ int maxFieldPos = numKeyFields / 2;
+ for (int j = 0; j < maxFieldPos; j++) {
+ int k = maxFieldPos + j;
+ int firstValue = rnd.nextInt() % maxValue;
+ int secondValue;
+ do {
+ secondValue = rnd.nextInt() % maxValue;
+ } while (secondValue < firstValue);
+ fieldValues[j] = firstValue;
+ fieldValues[k] = secondValue;
+ }
+ }
+
+ @Override
+ protected void setIntPayloadFields(int[] fieldValues, int numKeyFields, int numFields) {
+ for (int j = numKeyFields; j < numFields; j++) {
+ fieldValues[j] = intPayloadValue++;
+ }
+ }
+
+ @Override
+ protected Collection createCheckTuplesCollection() {
+ return new ArrayList<RTreeCheckTuple>();
+ }
+
+ @Override
+ protected ArrayTupleBuilder createDeleteTupleBuilder(ITreeIndexTestContext ctx) {
+ return new ArrayTupleBuilder(ctx.getFieldCount());
+ }
+
+ @Override
+ protected boolean checkDiskOrderScanResult(ITupleReference tuple, CheckTuple checkTuple, ITreeIndexTestContext ctx)
+ throws HyracksDataException {
+ return ctx.getCheckTuples().contains(checkTuple);
+ }
+}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestJobletContext.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestJobletContext.java
index 4f84466..7612db9 100644
--- a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestJobletContext.java
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestJobletContext.java
@@ -29,27 +29,26 @@
import edu.uci.ics.hyracks.control.nc.io.WorkspaceFileFactory;
public class TestJobletContext implements IHyracksJobletContext {
+ private final int frameSize;
private final INCApplicationContext appContext;
private JobId jobId;
private WorkspaceFileFactory fileFactory;
- public TestJobletContext(INCApplicationContext appContext, JobId jobId) throws HyracksException {
+ public TestJobletContext(int frameSize, INCApplicationContext appContext, JobId jobId) throws HyracksException {
+ this.frameSize = frameSize;
this.appContext = appContext;
this.jobId = jobId;
fileFactory = new WorkspaceFileFactory(this, (IOManager) getIOManager());
}
- @Override
public ByteBuffer allocateFrame() {
- return appContext.getRootContext().allocateFrame();
+ return ByteBuffer.allocate(frameSize);
}
- @Override
public int getFrameSize() {
- return appContext.getRootContext().getFrameSize();
+ return frameSize;
}
- @Override
public IIOManager getIOManager() {
return appContext.getRootContext().getIOManager();
}
@@ -88,4 +87,9 @@
public JobId getJobId() {
return jobId;
}
+
+ @Override
+ public Object getGlobalJobData() {
+ return null;
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestNCApplicationContext.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestNCApplicationContext.java
index 1b95ccb..0bd872c 100644
--- a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestNCApplicationContext.java
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestNCApplicationContext.java
@@ -18,6 +18,7 @@
import edu.uci.ics.hyracks.api.application.INCApplicationContext;
import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
+import edu.uci.ics.hyracks.api.messages.IMessageBroker;
public class TestNCApplicationContext implements INCApplicationContext {
private final IHyracksRootContext rootCtx;
@@ -60,4 +61,22 @@
public Object getApplicationObject() {
return appObject;
}
+
+ @Override
+ public String getApplicationName() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public void setMessageBroker(IMessageBroker staticticsConnector) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public IMessageBroker getMessageBroker() {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestRootContext.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestRootContext.java
index 93409ef..e195036 100644
--- a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestRootContext.java
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestRootContext.java
@@ -15,7 +15,6 @@
package edu.uci.ics.hyracks.test.support;
import java.io.File;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
@@ -27,27 +26,15 @@
import edu.uci.ics.hyracks.control.nc.io.IOManager;
public class TestRootContext implements IHyracksRootContext {
- private int frameSize;
private IOManager ioManager;
- public TestRootContext(int frameSize) throws HyracksException {
- this.frameSize = frameSize;
+ public TestRootContext() throws HyracksException {
List<IODeviceHandle> devices = new ArrayList<IODeviceHandle>();
devices.add(new IODeviceHandle(new File(System.getProperty("java.io.tmpdir")), "."));
ioManager = new IOManager(devices, Executors.newCachedThreadPool());
}
@Override
- public ByteBuffer allocateFrame() {
- return ByteBuffer.allocate(frameSize);
- }
-
- @Override
- public int getFrameSize() {
- return frameSize;
- }
-
- @Override
public IIOManager getIOManager() {
return ioManager;
}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerComponentHolder.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerComponentHolder.java
index d006f42..ccbe862 100644
--- a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerComponentHolder.java
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerComponentHolder.java
@@ -14,7 +14,15 @@
*/
package edu.uci.ics.hyracks.test.support;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executors;
+
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.io.IODeviceHandle;
+import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexRegistry;
import edu.uci.ics.hyracks.storage.common.buffercache.BufferCache;
@@ -31,7 +39,8 @@
private static IBufferCache bufferCache;
private static IFileMapProvider fileMapProvider;
private static IndexRegistry<IIndex> indexRegistry;
-
+ private static IOManager ioManager;
+
private static int pageSize;
private static int numPages;
private static int maxOpenFiles;
@@ -69,4 +78,13 @@
}
return indexRegistry;
}
+
+ public synchronized static IOManager getIOManager() throws HyracksException {
+ if (ioManager == null) {
+ List<IODeviceHandle> devices = new ArrayList<IODeviceHandle>();
+ devices.add(new IODeviceHandle(new File(System.getProperty("java.io.tmpdir")), "iodev_test_wa"));
+ ioManager = new IOManager(devices, Executors.newCachedThreadPool());
+ }
+ return ioManager;
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestTaskContext.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestTaskContext.java
index e1743a1..c122b25 100644
--- a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestTaskContext.java
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestTaskContext.java
@@ -19,8 +19,7 @@
import edu.uci.ics.hyracks.api.context.IHyracksJobletContext;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
-import edu.uci.ics.hyracks.api.dataflow.TaskId;
-import edu.uci.ics.hyracks.api.dataflow.state.ITaskState;
+import edu.uci.ics.hyracks.api.dataflow.state.IStateObject;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.api.io.FileReference;
@@ -31,11 +30,11 @@
import edu.uci.ics.hyracks.control.nc.io.WorkspaceFileFactory;
public class TestTaskContext implements IHyracksTaskContext {
- private final IHyracksJobletContext jobletContext;
+ private final TestJobletContext jobletContext;
private final TaskAttemptId taskId;
private WorkspaceFileFactory fileFactory;
- public TestTaskContext(IHyracksJobletContext jobletContext, TaskAttemptId taskId) throws HyracksException {
+ public TestTaskContext(TestJobletContext jobletContext, TaskAttemptId taskId) throws HyracksException {
this.jobletContext = jobletContext;
this.taskId = taskId;
fileFactory = new WorkspaceFileFactory(this, (IOManager) getIOManager());
@@ -92,12 +91,18 @@
}
@Override
- public void setTaskState(ITaskState taskState) {
+ public void setStateObject(IStateObject taskState) {
}
@Override
- public ITaskState getTaskState(TaskId taskId) {
+ public IStateObject getStateObject(Object id) {
return null;
}
+
+ @Override
+ public void sendApplicationMessageToCC(byte[] message, String nodeId) throws Exception {
+ // TODO Auto-generated method stub
+
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestUtils.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestUtils.java
index 5d488c1..71da634 100644
--- a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestUtils.java
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestUtils.java
@@ -15,7 +15,6 @@
package edu.uci.ics.hyracks.test.support;
import edu.uci.ics.hyracks.api.application.INCApplicationContext;
-import edu.uci.ics.hyracks.api.context.IHyracksJobletContext;
import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.ActivityId;
@@ -28,9 +27,9 @@
public class TestUtils {
public static IHyracksTaskContext create(int frameSize) {
try {
- IHyracksRootContext rootCtx = new TestRootContext(frameSize);
+ IHyracksRootContext rootCtx = new TestRootContext();
INCApplicationContext appCtx = new TestNCApplicationContext(rootCtx, null);
- IHyracksJobletContext jobletCtx = new TestJobletContext(appCtx, new JobId(0));
+ TestJobletContext jobletCtx = new TestJobletContext(frameSize, appCtx, new JobId(0));
TaskAttemptId tid = new TaskAttemptId(new TaskId(new ActivityId(new OperatorDescriptorId(0), 0), 0), 0);
IHyracksTaskContext taskCtx = new TestTaskContext(jobletCtx, tid);
return taskCtx;
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/pom.xml b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/pom.xml
index 6655844..607d13a 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/pom.xml
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/pom.xml
@@ -33,12 +33,6 @@
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
- <artifactId>hyracks-control-nc</artifactId>
- <version>0.2.1-SNAPSHOT</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-btree</artifactId>
<version>0.2.1-SNAPSHOT</version>
<type>jar</type>
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeBulkLoadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeBulkLoadTest.java
new file mode 100644
index 0000000..11c47c7
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeBulkLoadTest.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexBulkLoadTest;
+import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
+
+@SuppressWarnings("rawtypes")
+public class BTreeBulkLoadTest extends OrderedIndexBulkLoadTest {
+
+ public BTreeBulkLoadTest() {
+ super(BTreeTestHarness.LEAF_FRAMES_TO_TEST, 1);
+ }
+
+ private final BTreeTestHarness harness = new BTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
+ BTreeLeafFrameType leafType) throws Exception {
+ return BTreeTestContext.create(harness.getBufferCache(), harness.getBTreeFileId(), fieldSerdes, numKeys,
+ leafType);
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeDeleteTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeDeleteTest.java
new file mode 100644
index 0000000..0205540
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeDeleteTest.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexDeleteTest;
+import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
+
+@SuppressWarnings("rawtypes")
+public class BTreeDeleteTest extends OrderedIndexDeleteTest {
+
+ public BTreeDeleteTest() {
+ super(BTreeTestHarness.LEAF_FRAMES_TO_TEST);
+ }
+
+ private final BTreeTestHarness harness = new BTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
+ BTreeLeafFrameType leafType) throws Exception {
+ return BTreeTestContext.create(harness.getBufferCache(), harness.getBTreeFileId(), fieldSerdes, numKeys,
+ leafType);
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java
index 5bb86b9..f4f8b12 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java
@@ -15,610 +15,38 @@
package edu.uci.ics.hyracks.storage.am.btree;
-import java.util.Random;
-import java.util.logging.Level;
+import org.junit.After;
+import org.junit.Before;
-import org.junit.Test;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
-import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
-import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexExamplesTest;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
-import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
-import edu.uci.ics.hyracks.storage.am.btree.util.AbstractBTreeTest;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-@SuppressWarnings("rawtypes")
-public class BTreeExamplesTest extends AbstractBTreeTest {
+public class BTreeExamplesTest extends OrderedIndexExamplesTest {
+ private final BTreeTestHarness harness = new BTreeTestHarness();
- /**
- * Fixed-Length Key,Value Example.
- *
- * Create a BTree with one fixed-length key field and one fixed-length value
- * field. Fill BTree with random values using insertions (not bulk load).
- * Perform scans and range search.
- */
- @Test
- public void fixedLengthKeyValueExample() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Fixed-Length Key,Value Example.");
- }
-
- // Declare fields.
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = IntegerPointable.TYPE_TRAITS;
- typeTraits[1] = IntegerPointable.TYPE_TRAITS;
- // Declare field serdes.
- ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
-
- // Declare keys.
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
-
- BTree btree = BTreeUtils
- .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
-
- long start = System.currentTimeMillis();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Inserting into tree...");
- }
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- ArrayTupleReference tuple = new ArrayTupleReference();
- ITreeIndexAccessor indexAccessor = btree.createAccessor();
- int numInserts = 10000;
- for (int i = 0; i < numInserts; i++) {
- int f0 = rnd.nextInt() % numInserts;
- int f1 = 5;
- TupleUtils.createIntegerTuple(tb, tuple, f0, f1);
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("Inserting " + i + " : " + f0 + " " + f1);
- }
- }
- try {
- indexAccessor.insert(tuple);
- } catch (TreeIndexException e) {
- }
- }
- long end = System.currentTimeMillis();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
- }
-
- orderedScan(btree, indexAccessor, fieldSerdes);
- diskOrderScan(btree, indexAccessor, fieldSerdes);
-
- // Build low key.
- ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(keyFieldCount);
- ArrayTupleReference lowKey = new ArrayTupleReference();
- TupleUtils.createIntegerTuple(lowKeyTb, lowKey, -1000);
-
- // Build high key.
- ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(keyFieldCount);
- ArrayTupleReference highKey = new ArrayTupleReference();
- TupleUtils.createIntegerTuple(highKeyTb, highKey, 1000);
-
- rangeSearch(btree, indexAccessor, fieldSerdes, lowKey, highKey);
-
- btree.close();
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
}
- /**
- * Composite Key Example (Non-Unique B-Tree).
- *
- * Create a BTree with two fixed-length key fields and one fixed-length
- * value field. Fill BTree with random values using insertions (not bulk
- * load) Perform scans and range search.
- */
- @Test
- public void twoFixedLengthKeysOneFixedLengthValueExample() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Composite Key Test");
- }
-
- // Declare fields.
- int fieldCount = 3;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = IntegerPointable.TYPE_TRAITS;
- typeTraits[1] = IntegerPointable.TYPE_TRAITS;
- typeTraits[2] = IntegerPointable.TYPE_TRAITS;
- // Declare field serdes.
- ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
-
- // declare keys
- int keyFieldCount = 2;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
- cmps[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
-
- BTree btree = BTreeUtils
- .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
-
- long start = System.currentTimeMillis();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Inserting into tree...");
- }
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- ArrayTupleReference tuple = new ArrayTupleReference();
- ITreeIndexAccessor indexAccessor = btree.createAccessor();
- int numInserts = 10000;
- for (int i = 0; i < 10000; i++) {
- int f0 = rnd.nextInt() % 2000;
- int f1 = rnd.nextInt() % 1000;
- int f2 = 5;
- TupleUtils.createIntegerTuple(tb, tuple, f0, f1, f2);
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("Inserting " + i + " : " + f0 + " " + f1 + " " + f2);
- }
- }
- try {
- indexAccessor.insert(tuple);
- } catch (TreeIndexException e) {
- }
- }
- long end = System.currentTimeMillis();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
- }
-
- orderedScan(btree, indexAccessor, fieldSerdes);
- diskOrderScan(btree, indexAccessor, fieldSerdes);
-
- // Build low key.
- ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
- ArrayTupleReference lowKey = new ArrayTupleReference();
- TupleUtils.createIntegerTuple(lowKeyTb, lowKey, -3);
-
- // Build high key.
- ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
- ArrayTupleReference highKey = new ArrayTupleReference();
- TupleUtils.createIntegerTuple(highKeyTb, highKey, 3);
-
- // Prefix-Range search in [-3, 3]
- rangeSearch(btree, indexAccessor, fieldSerdes, lowKey, highKey);
-
- btree.close();
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
}
-
- /**
- * Variable-Length Example. Create a BTree with one variable-length key
- * field and one variable-length value field. Fill BTree with random values
- * using insertions (not bulk load) Perform ordered scans and range search.
- */
- @Test
- public void varLenKeyValueExample() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Variable-Length Key,Value Example");
- }
-
- // Declare fields.
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
- typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
- // Declare field serdes.
- ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
-
- // Declare keys.
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY).createBinaryComparator();
-
- BTree btree = BTreeUtils
- .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
-
- long start = System.currentTimeMillis();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Inserting into tree...");
- }
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- ArrayTupleReference tuple = new ArrayTupleReference();
- ITreeIndexAccessor indexAccessor = btree.createAccessor();
- // Max string length to be generated.
- int maxLength = 10;
- int numInserts = 10000;
- for (int i = 0; i < 10000; i++) {
- String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
- String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
- TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("Inserting " + f0 + " " + f1);
- }
- }
- try {
- indexAccessor.insert(tuple);
- } catch (TreeIndexException e) {
- }
- }
- long end = System.currentTimeMillis();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
- }
-
- orderedScan(btree, indexAccessor, fieldSerdes);
- diskOrderScan(btree, indexAccessor, fieldSerdes);
-
- // Build low key.
- ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
- ArrayTupleReference lowKey = new ArrayTupleReference();
- TupleUtils.createTuple(lowKeyTb, lowKey, fieldSerdes, "cbf");
-
- // Build high key.
- ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
- ArrayTupleReference highKey = new ArrayTupleReference();
- TupleUtils.createTuple(highKeyTb, highKey, fieldSerdes, "cc7");
-
- rangeSearch(btree, indexAccessor, fieldSerdes, lowKey, highKey);
-
- btree.close();
+
+ protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories) throws TreeIndexException {
+ return BTreeUtils.createBTree(harness.getBufferCache(), harness.getOpCallback(), typeTraits, cmpFactories,
+ BTreeLeafFrameType.REGULAR_NSM);
}
-
- /**
- * Deletion Example.
- *
- * Create a BTree with one variable-length key field and one variable-length
- * value field. Fill B-tree with random values using insertions, then delete
- * entries one-by-one. Repeat procedure a few times on same BTree.
- */
- @Test
- public void deleteExample() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Deletion Example");
- }
-
- // Declare fields.
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
- typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
- // Declare field serdes.
- ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
-
- // Declare keys.
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY).createBinaryComparator();
-
- BTree btree = BTreeUtils
- .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
-
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- ArrayTupleReference tuple = new ArrayTupleReference();
- ITreeIndexAccessor indexAccessor = btree.createAccessor();
- // Max string length to be generated.
- int runs = 3;
- for (int run = 0; run < runs; run++) {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Deletion example run: " + (run + 1) + "/" + runs);
- LOGGER.info("Inserting into tree...");
- }
- int maxLength = 10;
- int ins = 10000;
- String[] f0s = new String[ins];
- String[] f1s = new String[ins];
- int insDone = 0;
- int[] insDoneCmp = new int[ins];
- for (int i = 0; i < ins; i++) {
- String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
- String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
- TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
- f0s[i] = f0;
- f1s[i] = f1;
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("Inserting " + i);
- }
- }
- try {
- indexAccessor.insert(tuple);
- insDone++;
- } catch (TreeIndexException e) {
- }
- insDoneCmp[i] = insDone;
- }
-
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Deleting from tree...");
- }
- int delDone = 0;
- for (int i = 0; i < ins; i++) {
- TupleUtils.createTuple(tb, tuple, fieldSerdes, f0s[i], f1s[i]);
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("Deleting " + i);
- }
- }
- try {
- indexAccessor.delete(tuple);
- delDone++;
- } catch (TreeIndexException e) {
- }
- if (insDoneCmp[i] != delDone) {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("INCONSISTENT STATE, ERROR IN DELETION EXAMPLE.");
- LOGGER.info("INSDONECMP: " + insDoneCmp[i] + " " + delDone);
- }
- break;
- }
- }
- if (insDone != delDone) {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("ERROR! INSDONE: " + insDone + " DELDONE: " + delDone);
- }
- break;
- }
- }
- btree.close();
+
+ protected int getIndexFileId() {
+ return harness.getBTreeFileId();
}
-
- /**
- * Update example.
- *
- * Create a BTree with one variable-length key field and one variable-length
- * value field. Fill B-tree with random values using insertions, then update
- * entries one-by-one. Repeat procedure a few times on same BTree.
- */
- @Test
- public void updateExample() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Update example");
- }
-
- // Declare fields.
- int fieldCount = 2;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
- typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
- // Declare field serdes.
- ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
-
- // Declare keys.
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY).createBinaryComparator();
-
- BTree btree = BTreeUtils
- .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
-
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Inserting into tree...");
- }
- ITreeIndexAccessor indexAccessor = btree.createAccessor();
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- ArrayTupleReference tuple = new ArrayTupleReference();
- int maxLength = 10;
- int ins = 10000;
- String[] keys = new String[10000];
- for (int i = 0; i < ins; i++) {
- String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
- String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
- TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
- keys[i] = f0;
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("Inserting " + i);
- }
- }
- try {
- indexAccessor.insert(tuple);
- } catch (TreeIndexException e) {
- }
- }
- // Print before doing any updates.
- orderedScan(btree, indexAccessor, fieldSerdes);
-
- int runs = 3;
- for (int run = 0; run < runs; run++) {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Update test run: " + (run + 1) + "/" + runs);
- LOGGER.info("Updating BTree");
- }
- for (int i = 0; i < ins; i++) {
- // Generate a new random value for f1.
- String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
- TupleUtils.createTuple(tb, tuple, fieldSerdes, keys[i], f1);
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("UPDATING " + i);
- }
- }
- try {
- indexAccessor.update(tuple);
- } catch (TreeIndexException e) {
- }
- }
- // Do another scan after a round of updates.
- orderedScan(btree, indexAccessor, fieldSerdes);
- }
- btree.close();
- }
-
- /**
- * Bulk load example.
- *
- * Load a tree with 100,000 tuples. BTree has a composite key to "simulate"
- * non-unique index creation.
- *
- */
- @Test
- public void bulkLoadExample() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Bulk load example");
- }
- // Declare fields.
- int fieldCount = 3;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = IntegerPointable.TYPE_TRAITS;
- typeTraits[1] = IntegerPointable.TYPE_TRAITS;
- typeTraits[2] = IntegerPointable.TYPE_TRAITS;
- // Declare field serdes.
- ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
-
- // declare keys
- int keyFieldCount = 2;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
- cmps[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
-
- BTree btree = BTreeUtils
- .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
-
- // Load sorted records.
- int ins = 100000;
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Bulk loading " + ins + " tuples");
- }
- long start = System.currentTimeMillis();
- IIndexBulkLoadContext bulkLoadCtx = btree.beginBulkLoad(0.7f);
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- ArrayTupleReference tuple = new ArrayTupleReference();
- for (int i = 0; i < ins; i++) {
- TupleUtils.createIntegerTuple(tb, tuple, i, i, 5);
- btree.bulkLoadAddTuple(tuple, bulkLoadCtx);
- }
- btree.endBulkLoad(bulkLoadCtx);
- long end = System.currentTimeMillis();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(ins + " tuples loaded in " + (end - start) + "ms");
- }
-
- ITreeIndexAccessor indexAccessor = btree.createAccessor();
-
- // Build low key.
- ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
- ArrayTupleReference lowKey = new ArrayTupleReference();
- TupleUtils.createIntegerTuple(lowKeyTb, lowKey, 44444);
-
- // Build high key.
- ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
- ArrayTupleReference highKey = new ArrayTupleReference();
- TupleUtils.createIntegerTuple(highKeyTb, highKey, 44500);
-
- // Prefix-Range search in [44444, 44500]
- rangeSearch(btree, indexAccessor, fieldSerdes, lowKey, highKey);
-
- btree.close();
- }
-
- private void orderedScan(BTree btree, ITreeIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes)
- throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Ordered Scan:");
- }
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
- ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(leafFrame, false);
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- indexAccessor.search(scanCursor, nullPred);
- try {
- while (scanCursor.hasNext()) {
- scanCursor.next();
- ITupleReference frameTuple = scanCursor.getTuple();
- String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(rec);
- }
- }
- } finally {
- scanCursor.close();
- }
- }
-
- private void diskOrderScan(BTree btree, ITreeIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes)
- throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Disk-Order Scan:");
- }
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
- TreeDiskOrderScanCursor diskOrderCursor = new TreeDiskOrderScanCursor(leafFrame);
- indexAccessor.diskOrderScan(diskOrderCursor);
- try {
- while (diskOrderCursor.hasNext()) {
- diskOrderCursor.next();
- ITupleReference frameTuple = diskOrderCursor.getTuple();
- String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(rec);
- }
- }
- } finally {
- diskOrderCursor.close();
- }
- }
-
- private void rangeSearch(BTree btree, ITreeIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes,
- ITupleReference lowKey, ITupleReference highKey) throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- String lowKeyString = TupleUtils.printTuple(lowKey, fieldSerdes);
- String highKeyString = TupleUtils.printTuple(highKey, fieldSerdes);
- LOGGER.info("Range-Search in: [" + lowKeyString + ", " + highKeyString + "]");
- }
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
- MultiComparator lowKeySearchCmp = BTreeUtils.getSearchMultiComparator(btree.getMultiComparator(), lowKey);
- MultiComparator highKeySearchCmp = BTreeUtils.getSearchMultiComparator(btree.getMultiComparator(), highKey);
- ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame, false);
- RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, true, true, lowKeySearchCmp,
- highKeySearchCmp);
- indexAccessor.search(rangeCursor, rangePred);
- try {
- while (rangeCursor.hasNext()) {
- rangeCursor.next();
- ITupleReference frameTuple = rangeCursor.getTuple();
- String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(rec);
- }
- }
- } finally {
- rangeCursor.close();
- }
- }
-
- public static String randomString(int length, Random random) {
- String s = Long.toHexString(Double.doubleToLongBits(random.nextDouble()));
- StringBuilder strBuilder = new StringBuilder();
- for (int i = 0; i < s.length() && i < length; i++) {
- strBuilder.append(s.charAt(Math.abs(random.nextInt()) % s.length()));
- }
- return strBuilder.toString();
- }
-}
\ No newline at end of file
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeInsertTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeInsertTest.java
new file mode 100644
index 0000000..0b6cf4d
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeInsertTest.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexInsertTest;
+import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
+
+/**
+ * Tests the BTree insert operation with strings and integer fields using
+ * various numbers of key and payload fields.
+ *
+ * Each tests first fills a BTree with randomly generated tuples. We compare the
+ * following operations against expected results: 1. Point searches for all
+ * tuples. 2. Ordered scan. 3. Disk-order scan. 4. Range search (and prefix
+ * search for composite keys).
+ *
+ */
+@SuppressWarnings("rawtypes")
+public class BTreeInsertTest extends OrderedIndexInsertTest {
+
+ public BTreeInsertTest() {
+ super(BTreeTestHarness.LEAF_FRAMES_TO_TEST);
+ }
+
+ private final BTreeTestHarness harness = new BTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
+ BTreeLeafFrameType leafType) throws Exception {
+ return BTreeTestContext.create(harness.getBufferCache(), harness.getBTreeFileId(), fieldSerdes, numKeys,
+ leafType);
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/RangeSearchCursorTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeSearchCursorTest.java
similarity index 78%
rename from hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/RangeSearchCursorTest.java
rename to hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeSearchCursorTest.java
index 4031e7f..4003cf1 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/RangeSearchCursorTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeSearchCursorTest.java
@@ -55,11 +55,12 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.am.common.util.IndexUtils;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-public class RangeSearchCursorTest extends AbstractBTreeTest {
+public class BTreeSearchCursorTest extends AbstractBTreeTest {
// Declare fields
int fieldCount = 2;
ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
@@ -83,22 +84,23 @@
LOGGER.info("TESTING RANGE SEARCH CURSOR ON UNIQUE INDEX");
}
+ IBufferCache bufferCache = harness.getBufferCache();
+ int btreeFileId = harness.getBTreeFileId();
+
// declare keys
int keyFieldCount = 1;
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- MultiComparator cmp = IndexUtils.createMultiComparator(cmpFactories);
-
ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
- BTree btree = new BTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ BTree btree = new BTree(bufferCache, NoOpOperationCallback.INSTANCE, fieldCount, cmpFactories, freePageManager, interiorFrameFactory, leafFrameFactory);
btree.create(btreeFileId);
btree.open(btreeFileId);
@@ -140,16 +142,10 @@
int maxSearchKey = 100;
// forward searches
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, false, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, true, false);
-
- // backward searches
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, false, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, false, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, false, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false);
btree.close();
}
@@ -160,23 +156,24 @@
LOGGER.info("TESTING RANGE SEARCH CURSOR ON NONUNIQUE INDEX");
}
+ IBufferCache bufferCache = harness.getBufferCache();
+ int btreeFileId = harness.getBTreeFileId();
+
// declare keys
int keyFieldCount = 2;
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- MultiComparator cmp = IndexUtils.createMultiComparator(cmpFactories);
-
ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
- BTree btree = new BTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ BTree btree = new BTree(bufferCache, NoOpOperationCallback.INSTANCE, fieldCount, cmpFactories, freePageManager, interiorFrameFactory, leafFrameFactory);
btree.create(btreeFileId);
btree.open(btreeFileId);
@@ -215,16 +212,10 @@
int maxSearchKey = 100;
// forward searches
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, false, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, true, false);
-
- // backward searches
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, false, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, false, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, false, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false);
btree.close();
}
@@ -235,23 +226,24 @@
LOGGER.info("TESTING RANGE SEARCH CURSOR ON NONUNIQUE FIELD-PREFIX COMPRESSED INDEX");
}
+ IBufferCache bufferCache = harness.getBufferCache();
+ int btreeFileId = harness.getBTreeFileId();
+
// declare keys
int keyFieldCount = 2;
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- MultiComparator cmp = IndexUtils.createMultiComparator(cmpFactories);
-
ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
- BTree btree = new BTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ BTree btree = new BTree(bufferCache, NoOpOperationCallback.INSTANCE, fieldCount, cmpFactories, freePageManager, interiorFrameFactory, leafFrameFactory);
btree.create(btreeFileId);
btree.open(btreeFileId);
@@ -290,22 +282,16 @@
int maxSearchKey = 100;
// forward searches
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, false, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, true, false);
-
- // backward searches
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, false, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, false, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, false, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false);
btree.close();
}
- public RangePredicate createRangePredicate(int lk, int hk, boolean isForward, boolean lowKeyInclusive,
- boolean highKeyInclusive, MultiComparator cmp) throws HyracksDataException {
+ public RangePredicate createRangePredicate(int lk, int hk, boolean lowKeyInclusive,
+ boolean highKeyInclusive) throws HyracksDataException {
// create tuplereferences for search keys
ITupleReference lowKey = TupleUtils.createIntegerTuple(lk);
@@ -315,13 +301,13 @@
searchCmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
MultiComparator searchCmp = new MultiComparator(searchCmps);
- RangePredicate rangePred = new RangePredicate(isForward, lowKey, highKey, lowKeyInclusive, highKeyInclusive,
+ RangePredicate rangePred = new RangePredicate(lowKey, highKey, lowKeyInclusive, highKeyInclusive,
searchCmp, searchCmp);
return rangePred;
}
public void getExpectedResults(ArrayList<Integer> expectedResults, ArrayList<Integer> keys, int lk, int hk,
- boolean isForward, boolean lowKeyInclusive, boolean highKeyInclusive) {
+ boolean lowKeyInclusive, boolean highKeyInclusive) {
// special cases
if (lk == hk && (!lowKeyInclusive || !highKeyInclusive))
@@ -329,35 +315,21 @@
if (lk > hk)
return;
- if (isForward) {
- for (int i = 0; i < keys.size(); i++) {
- if ((lk == keys.get(i) && lowKeyInclusive) || (hk == keys.get(i) && highKeyInclusive)) {
- expectedResults.add(keys.get(i));
- continue;
- }
-
- if (lk < keys.get(i) && hk > keys.get(i)) {
- expectedResults.add(keys.get(i));
- continue;
- }
+ for (int i = 0; i < keys.size(); i++) {
+ if ((lk == keys.get(i) && lowKeyInclusive) || (hk == keys.get(i) && highKeyInclusive)) {
+ expectedResults.add(keys.get(i));
+ continue;
}
- } else {
- for (int i = keys.size() - 1; i >= 0; i--) {
- if ((lk == keys.get(i) && lowKeyInclusive) || (hk == keys.get(i) && highKeyInclusive)) {
- expectedResults.add(keys.get(i));
- continue;
- }
- if (lk < keys.get(i) && hk > keys.get(i)) {
- expectedResults.add(keys.get(i));
- continue;
- }
+ if (lk < keys.get(i) && hk > keys.get(i)) {
+ expectedResults.add(keys.get(i));
+ continue;
}
}
}
public boolean performSearches(ArrayList<Integer> keys, BTree btree, IBTreeLeafFrame leafFrame,
- IBTreeInteriorFrame interiorFrame, int minKey, int maxKey, boolean isForward, boolean lowKeyInclusive,
+ IBTreeInteriorFrame interiorFrame, int minKey, int maxKey, boolean lowKeyInclusive,
boolean highKeyInclusive, boolean printExpectedResults) throws Exception {
ArrayList<Integer> results = new ArrayList<Integer>();
@@ -373,8 +345,8 @@
int highKey = j;
ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame, false);
- RangePredicate rangePred = createRangePredicate(lowKey, highKey, isForward, lowKeyInclusive,
- highKeyInclusive, btree.getMultiComparator());
+ RangePredicate rangePred = createRangePredicate(lowKey, highKey, lowKeyInclusive,
+ highKeyInclusive);
ITreeIndexAccessor indexAccessor = btree.createAccessor();
indexAccessor.search(rangeCursor, rangePred);
@@ -394,7 +366,7 @@
rangeCursor.close();
}
- getExpectedResults(expectedResults, keys, lowKey, highKey, isForward, lowKeyInclusive, highKeyInclusive);
+ getExpectedResults(expectedResults, keys, lowKey, highKey, lowKeyInclusive, highKeyInclusive);
if (printExpectedResults) {
if (expectedResults.size() > 0) {
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StatsTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
similarity index 91%
rename from hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StatsTest.java
rename to hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
index 1ae80d6..c33b4e9 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StatsTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
@@ -36,9 +36,8 @@
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.am.common.util.IndexUtils;
import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexBufferCacheWarmup;
import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexStats;
import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexStatsGatherer;
@@ -47,11 +46,8 @@
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
import edu.uci.ics.hyracks.test.support.TestUtils;
-public class StatsTest extends AbstractBTreeTest {
-
- // private static final int PAGE_SIZE = 256;
- // private static final int NUM_PAGES = 10;
- // private static final int PAGE_SIZE = 32768;
+@SuppressWarnings("rawtypes")
+public class BTreeStatsTest extends AbstractBTreeTest {
private static final int PAGE_SIZE = 4096;
private static final int NUM_PAGES = 1000;
private static final int MAX_OPEN_FILES = 10;
@@ -64,7 +60,7 @@
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
+ FileReference file = new FileReference(new File(harness.getFileName()));
bufferCache.createFile(file);
int fileId = fmp.lookupFileId(file);
bufferCache.openFile(fileId);
@@ -80,8 +76,6 @@
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- MultiComparator cmp = IndexUtils.createMultiComparator(cmpFactories);
-
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
@@ -91,9 +85,9 @@
IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
- BTree btree = new BTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ BTree btree = new BTree(bufferCache, NoOpOperationCallback.INSTANCE, fieldCount, cmpFactories, freePageManager, interiorFrameFactory, leafFrameFactory);
btree.create(fileId);
btree.open(fileId);
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeTestDriver.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeTestDriver.java
deleted file mode 100644
index 1daa273..0000000
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeTestDriver.java
+++ /dev/null
@@ -1,133 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.btree;
-
-import java.util.logging.Level;
-
-import org.junit.Test;
-
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
-import edu.uci.ics.hyracks.storage.am.btree.util.AbstractBTreeTest;
-
-@SuppressWarnings("rawtypes")
-public abstract class BTreeTestDriver extends AbstractBTreeTest {
-
- protected static final int numTuplesToInsert = 10000;
-
- protected abstract void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception;
- protected abstract String getTestOpName();
-
- @Test
- public void oneIntKeyAndValue() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("BTree " + getTestOpName() + " Test With One Int Key And Value.");
- }
-
- ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
- // Range search in [-1000, 1000]
- ITupleReference lowKey = TupleUtils.createIntegerTuple(-1000);
- ITupleReference highKey = TupleUtils.createIntegerTuple(1000);
-
- runTest(fieldSerdes, 1, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, null, null);
- runTest(fieldSerdes, 1, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, null, null);
- }
-
- @Test
- public void twoIntKeys() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("BTree " + getTestOpName() + " Test With Two Int Keys.");
- }
-
- ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
-
- // Range search in [50 0, 50 500]
- ITupleReference lowKey = TupleUtils.createIntegerTuple(50, 0);
- ITupleReference highKey = TupleUtils.createIntegerTuple(50, 500);
-
- // Prefix range search in [50, 50]
- ITupleReference prefixLowKey = TupleUtils.createIntegerTuple(50);
- ITupleReference prefixHighKey = TupleUtils.createIntegerTuple(50);
-
- runTest(fieldSerdes, 2, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
- runTest(fieldSerdes, 2, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
- }
-
- @Test
- public void twoIntKeysAndValues() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("BTree " + getTestOpName() + " Test With Two Int Keys And Values.");
- }
-
- ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
-
- // Range search in [50 100, 100 100]
- ITupleReference lowKey = TupleUtils.createIntegerTuple(-100, -100);
- ITupleReference highKey = TupleUtils.createIntegerTuple(100, 100);
-
- // Prefix range search in [50, 50]
- ITupleReference prefixLowKey = TupleUtils.createIntegerTuple(50);
- ITupleReference prefixHighKey = TupleUtils.createIntegerTuple(50);
-
- runTest(fieldSerdes, 2, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
- runTest(fieldSerdes, 2, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
- }
-
- @Test
- public void oneStringKeyAndValue() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("BTree " + getTestOpName() + " Test With One String Key And Value.");
- }
-
- ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE };
-
- // Range search in ["cbf", cc7"]
- ITupleReference lowKey = TupleUtils.createTuple(fieldSerdes, "cbf");
- ITupleReference highKey = TupleUtils.createTuple(fieldSerdes, "cc7");
-
- runTest(fieldSerdes, 1, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, null, null);
- runTest(fieldSerdes, 1, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, null, null);
- }
-
- @Test
- public void twoStringKeys() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("BTree " + getTestOpName() + " Test With Two String Keys.");
- }
-
- ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE };
-
- // Range search in ["cbf", "ddd", cc7", "eee"]
- ITupleReference lowKey = TupleUtils.createTuple(fieldSerdes, "cbf", "ddd");
- ITupleReference highKey = TupleUtils.createTuple(fieldSerdes, "cc7", "eee");
-
- // Prefix range search in ["cbf", cc7"]
- ITupleReference prefixLowKey = TupleUtils.createTuple(fieldSerdes, "cbf");
- ITupleReference prefixHighKey = TupleUtils.createTuple(fieldSerdes, "cc7");
-
- runTest(fieldSerdes, 2, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
- runTest(fieldSerdes, 2, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
- }
-
- @Test
- public void twoStringKeysAndValues() throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("BTree " + getTestOpName() + " Test With Two String Keys And Values.");
- }
-
- ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE };
-
- // Range search in ["cbf", "ddd", cc7", "eee"]
- ITupleReference lowKey = TupleUtils.createTuple(fieldSerdes, "cbf", "ddd");
- ITupleReference highKey = TupleUtils.createTuple(fieldSerdes, "cc7", "eee");
-
- // Prefix range search in ["cbf", cc7"]
- ITupleReference prefixLowKey = TupleUtils.createTuple(fieldSerdes, "cbf");
- ITupleReference prefixHighKey = TupleUtils.createTuple(fieldSerdes, "cc7");
-
- runTest(fieldSerdes, 2, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
- runTest(fieldSerdes, 2, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
- }
-}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateSearchTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
similarity index 89%
rename from hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateSearchTest.java
rename to hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
index 5d43ae1..2b03a6a 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateSearchTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
@@ -30,15 +30,19 @@
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.am.common.util.IndexUtils;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-public class UpdateSearchTest extends AbstractBTreeTest {
+@SuppressWarnings("rawtypes")
+public class BTreeUpdateSearchTest extends AbstractBTreeTest {
// Update scan test on fixed-length tuples.
@Test
public void test01() throws Exception {
+ IBufferCache bufferCache = harness.getBufferCache();
+ int btreeFileId = harness.getBTreeFileId();
+
// declare fields
int fieldCount = 2;
ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
@@ -50,8 +54,6 @@
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- MultiComparator cmp = IndexUtils.createMultiComparator(cmpFactories);
-
ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
IntegerSerializerDeserializer.INSTANCE };
@@ -62,8 +64,8 @@
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
- BTree btree = new BTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
+ BTree btree = new BTree(bufferCache, NoOpOperationCallback.INSTANCE, fieldCount, cmpFactories, freePageManager, interiorFrameFactory, leafFrameFactory);
btree.create(btreeFileId);
btree.open(btreeFileId);
@@ -81,7 +83,7 @@
ITreeIndexAccessor indexAccessor = btree.createAccessor();
int numInserts = 10000;
- for (int i = 0; i < 10000; i++) {
+ for (int i = 0; i < numInserts; i++) {
int f0 = rnd.nextInt() % 10000;
int f1 = 5;
TupleUtils.createIntegerTuple(tb, insertTuple, f0, f1);
@@ -111,7 +113,7 @@
}
// Set the cursor to X latch nodes.
ITreeIndexCursor updateScanCursor = new BTreeRangeSearchCursor(leafFrame, true);
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
+ RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null);
indexAccessor.search(updateScanCursor, nullPred);
try {
while (updateScanCursor.hasNext()) {
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateTest.java
new file mode 100644
index 0000000..c3b56d5
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpdateTest.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexUpdateTest;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
+
+@SuppressWarnings("rawtypes")
+public class BTreeUpdateTest extends OrderedIndexUpdateTest {
+
+ public BTreeUpdateTest() {
+ super(BTreeTestHarness.LEAF_FRAMES_TO_TEST);
+ }
+
+ private final BTreeTestHarness harness = new BTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
+ BTreeLeafFrameType leafType) throws Exception {
+ return BTreeTestContext.create(harness.getBufferCache(), harness.getBTreeFileId(), fieldSerdes, numKeys,
+ leafType);
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpsertTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpsertTest.java
new file mode 100644
index 0000000..6e14607
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeUpsertTest.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
+
+/**
+ * Tests the BTree insert operation with strings and integer fields using
+ * various numbers of key and payload fields.
+ *
+ * Each tests first fills a BTree with randomly generated tuples. We compare the
+ * following operations against expected results: 1. Point searches for all
+ * tuples. 2. Ordered scan. 3. Disk-order scan. 4. Range search (and prefix
+ * search for composite keys).
+ *
+ */
+@SuppressWarnings("rawtypes")
+public class BTreeUpsertTest extends OrderedIndexUpsertTest {
+
+ public BTreeUpsertTest() {
+ super(BTreeTestHarness.LEAF_FRAMES_TO_TEST);
+ }
+
+ private final BTreeTestHarness harness = new BTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected OrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
+ BTreeLeafFrameType leafType) throws Exception {
+ return BTreeTestContext.create(harness.getBufferCache(), harness.getBTreeFileId(), fieldSerdes, numKeys,
+ leafType);
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BulkLoadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BulkLoadTest.java
deleted file mode 100644
index 49d08a3..0000000
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BulkLoadTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.btree;
-
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestUtils;
-
-@SuppressWarnings("rawtypes")
-public class BulkLoadTest extends BTreeTestDriver {
-
- @Override
- protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception {
- BTreeTestContext testCtx = BTreeTestUtils.createBTreeTestContext(bufferCache, btreeFileId, fieldSerdes, numKeys, leafType);
-
- // We assume all fieldSerdes are of the same type. Check the first one to determine which field types to generate.
- if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
- BTreeTestUtils.bulkLoadIntTuples(testCtx, numTuplesToInsert, rnd);
- } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
- BTreeTestUtils.bulkLoadStringTuples(testCtx, numTuplesToInsert, rnd);
- }
-
- BTreeTestUtils.checkPointSearches(testCtx);
- BTreeTestUtils.checkOrderedScan(testCtx);
- BTreeTestUtils.checkDiskOrderScan(testCtx);
- BTreeTestUtils.checkRangeSearch(testCtx, lowKey, highKey, true, true);
- if (prefixLowKey != null && prefixHighKey != null) {
- BTreeTestUtils.checkRangeSearch(testCtx, prefixLowKey, prefixHighKey, true, true);
- }
- }
-
- @Override
- protected String getTestOpName() {
- return "BulkLoad";
- }
-}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/DeleteTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/DeleteTest.java
deleted file mode 100644
index 2157adb..0000000
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/DeleteTest.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.btree;
-
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestUtils;
-
-@SuppressWarnings("rawtypes")
-public class DeleteTest extends BTreeTestDriver {
-
- private static final int numInsertRounds = 3;
- private static final int numDeleteRounds = 3;
-
- @Override
- protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception {
- BTreeTestContext testCtx = BTreeTestUtils.createBTreeTestContext(bufferCache, btreeFileId, fieldSerdes, numKeys, leafType);
- for (int i = 0; i < numInsertRounds; i++) {
-
- // We assume all fieldSerdes are of the same type. Check the first one to determine which field types to generate.
- if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
- BTreeTestUtils.insertIntTuples(testCtx, numTuplesToInsert, rnd);
- } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
- BTreeTestUtils.insertStringTuples(testCtx, numTuplesToInsert, rnd);
- }
-
- int numTuplesPerDeleteRound = (int)Math.ceil((float)testCtx.checkTuples.size() / (float)numDeleteRounds);
- for(int j = 0; j < numDeleteRounds; j++) {
- BTreeTestUtils.deleteTuples(testCtx, numTuplesPerDeleteRound, rnd);
-
- BTreeTestUtils.checkPointSearches(testCtx);
- BTreeTestUtils.checkOrderedScan(testCtx);
- BTreeTestUtils.checkDiskOrderScan(testCtx);
- BTreeTestUtils.checkRangeSearch(testCtx, lowKey, highKey, true, true);
- if (prefixLowKey != null && prefixHighKey != null) {
- BTreeTestUtils.checkRangeSearch(testCtx, prefixLowKey, prefixHighKey, true, true);
- }
- }
- }
- }
-
- @Override
- protected String getTestOpName() {
- return "Delete";
- }
-}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java
index e8aa00a..d61d16a 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java
@@ -45,9 +45,11 @@
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexUtils;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+@SuppressWarnings("rawtypes")
public class FieldPrefixNSMTest extends AbstractBTreeTest {
private static final int PAGE_SIZE = 32768; // 32K
@@ -55,6 +57,10 @@
private static final int MAX_OPEN_FILES = 10;
private static final int HYRACKS_FRAME_SIZE = 128;
+ public FieldPrefixNSMTest() {
+ super(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES, HYRACKS_FRAME_SIZE);
+ }
+
private ITupleReference createTuple(IHyracksTaskContext ctx, int f0, int f1, int f2, boolean print)
throws HyracksDataException {
if (print) {
@@ -67,7 +73,7 @@
FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
ArrayTupleBuilder tb = new ArrayTupleBuilder(3);
DataOutput dos = tb.getDataOutput();
-
+
ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
@@ -92,8 +98,8 @@
}
@Test
- public void test01() throws Exception {
-
+ public void test01() throws Exception {
+
// declare fields
int fieldCount = 3;
ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
@@ -116,6 +122,9 @@
Random rnd = new Random();
rnd.setSeed(50);
+ IBufferCache bufferCache = harness.getBufferCache();
+ int btreeFileId = harness.getBTreeFileId();
+ IHyracksTaskContext ctx = harness.getHyracksTaskContext();
ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(btreeFileId, 0), false);
try {
@@ -213,20 +222,4 @@
bufferCache.unpin(page);
}
}
-
- public int getPageSize() {
- return PAGE_SIZE;
- }
-
- public int getNumPages() {
- return NUM_PAGES;
- }
-
- public int getHyracksFrameSize() {
- return HYRACKS_FRAME_SIZE;
- }
-
- public int getMaxOpenFiles() {
- return MAX_OPEN_FILES;
- }
}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/InsertTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/InsertTest.java
deleted file mode 100644
index fa3f895..0000000
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/InsertTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.btree;
-
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestUtils;
-
-/**
- * Tests the BTree insert operation with strings and integer fields using
- * various numbers of key and payload fields.
- *
- * Each tests first fills a BTree with randomly generated tuples.
- * We compare the following operations against expected results:
- * 1. Point searches for all tuples.
- * 2. Ordered scan.
- * 3. Disk-order scan.
- * 4. Range search (and prefix search for composite keys).
- *
- */
-@SuppressWarnings("rawtypes")
-public class InsertTest extends BTreeTestDriver {
- @Override
- protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception {
- BTreeTestContext testCtx = BTreeTestUtils.createBTreeTestContext(bufferCache, btreeFileId, fieldSerdes, numKeys, leafType);
- // We assume all fieldSerdes are of the same type. Check the first one to determine which field types to generate.
- if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
- BTreeTestUtils.insertIntTuples(testCtx, numTuplesToInsert, rnd);
- } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
- BTreeTestUtils.insertStringTuples(testCtx, numTuplesToInsert, rnd);
- }
-
- BTreeTestUtils.checkPointSearches(testCtx);
- BTreeTestUtils.checkOrderedScan(testCtx);
- BTreeTestUtils.checkDiskOrderScan(testCtx);
-
- BTreeTestUtils.checkRangeSearch(testCtx, lowKey, highKey, true, true);
- if (prefixLowKey != null && prefixHighKey != null) {
- BTreeTestUtils.checkRangeSearch(testCtx, prefixLowKey, prefixHighKey, true, true);
- }
- testCtx.btree.close();
- }
-
- @Override
- protected String getTestOpName() {
- return "Insert";
- }
-}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java
index ac4133d..9ec64b5 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java
@@ -15,7 +15,6 @@
package edu.uci.ics.hyracks.storage.am.btree;
-import java.io.File;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;
@@ -24,21 +23,13 @@
import org.junit.Test;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.storage.am.btree.util.AbstractBTreeTest;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.storage.common.sync.LatchType;
-import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
-public class StorageManagerTest extends AbstractBTreeTest {
- private static final int PAGE_SIZE = 256;
- private static final int NUM_PAGES = 10;
- private static final int MAX_OPEN_FILES = 10;
- private static final int HYRACKS_FRAME_SIZE = 32768;
-
+public class StorageManagerTest extends AbstractBTreeTest {
public class PinnedLatchedPage {
public final ICachedPage page;
public final LatchType latch;
@@ -258,38 +249,11 @@
}
@Test
- public void oneThreadOneFileTest() throws Exception {
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- Thread worker = new Thread(new FileAccessWorker(0, bufferCache, FileAccessType.FTA_UNLATCHED, fileId, 10, 10,
- 100, 10, 0));
-
+ public void oneThreadOneFileTest() throws Exception {
+ Thread worker = new Thread(new FileAccessWorker(0,
+ harness.getBufferCache(), FileAccessType.FTA_UNLATCHED,
+ harness.getBTreeFileId(), 10, 10, 100, 10, 0));
worker.start();
-
worker.join();
-
- bufferCache.close();
- }
-
- public int getPageSize() {
- return PAGE_SIZE;
- }
-
- public int getNumPages() {
- return NUM_PAGES;
- }
-
- public int getHyracksFrameSize() {
- return HYRACKS_FRAME_SIZE;
- }
-
- public int getMaxOpenFiles() {
- return MAX_OPEN_FILES;
}
}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateTest.java
deleted file mode 100644
index b40539f..0000000
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.btree;
-
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
-import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestUtils;
-
-@SuppressWarnings("rawtypes")
-public class UpdateTest extends BTreeTestDriver {
- private static final int numUpdateRounds = 3;
-
- @Override
- protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception {
- // This is a noop because we can only update non-key fields.
- if (fieldSerdes.length == numKeys) {
- return;
- }
-
- BTreeTestContext testCtx = BTreeTestUtils.createBTreeTestContext(bufferCache, btreeFileId, fieldSerdes, numKeys, leafType);
-
- // We assume all fieldSerdes are of the same type. Check the first one to determine which field types to generate.
- if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
- BTreeTestUtils.insertIntTuples(testCtx, numTuplesToInsert, rnd);
- } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
- BTreeTestUtils.insertStringTuples(testCtx, numTuplesToInsert, rnd);
- }
-
- int numTuplesPerDeleteRound = (int)Math.ceil((float)testCtx.checkTuples.size() / (float)numUpdateRounds);
- for(int j = 0; j < numUpdateRounds; j++) {
- BTreeTestUtils.updateTuples(testCtx, numTuplesPerDeleteRound, rnd);
-
- BTreeTestUtils.checkPointSearches(testCtx);
- BTreeTestUtils.checkOrderedScan(testCtx);
- BTreeTestUtils.checkDiskOrderScan(testCtx);
- BTreeTestUtils.checkRangeSearch(testCtx, lowKey, highKey, true, true);
- if (prefixLowKey != null && prefixHighKey != null) {
- BTreeTestUtils.checkRangeSearch(testCtx, prefixLowKey, prefixHighKey, true, true);
- }
- }
- }
-
- @Override
- protected String getTestOpName() {
- return "Update";
- }
-}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java
new file mode 100644
index 0000000..596fa31
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/multithread/BTreeMultiThreadTest.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.multithread;
+
+import java.util.ArrayList;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexMultiThreadTest;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestHarness;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
+import edu.uci.ics.hyracks.storage.am.common.ITreeIndexTestWorkerFactory;
+import edu.uci.ics.hyracks.storage.am.common.TestWorkloadConf;
+import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+
+public class BTreeMultiThreadTest extends OrderedIndexMultiThreadTest {
+
+ private BTreeTestHarness harness = new BTreeTestHarness();
+
+ private BTreeTestWorkerFactory workerFactory = new BTreeTestWorkerFactory();
+
+ @Override
+ protected void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @Override
+ protected void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories) throws TreeIndexException {
+ return BTreeUtils.createBTree(harness.getBufferCache(), harness.getOpCallback(), typeTraits, cmpFactories, BTreeLeafFrameType.REGULAR_NSM);
+ }
+
+ @Override
+ protected ITreeIndexTestWorkerFactory getWorkerFactory() {
+ return workerFactory;
+ }
+
+ @Override
+ protected ArrayList<TestWorkloadConf> getTestWorkloadConf() {
+ ArrayList<TestWorkloadConf> workloadConfs = new ArrayList<TestWorkloadConf>();
+
+ // Insert only workload.
+ TestOperation[] insertOnlyOps = new TestOperation[] { TestOperation.INSERT };
+ workloadConfs.add(new TestWorkloadConf(insertOnlyOps, getUniformOpProbs(insertOnlyOps)));
+
+ // Inserts mixed with point searches and scans.
+ TestOperation[] insertSearchOnlyOps = new TestOperation[] { TestOperation.INSERT, TestOperation.POINT_SEARCH, TestOperation.SCAN, TestOperation.DISKORDER_SCAN };
+ workloadConfs.add(new TestWorkloadConf(insertSearchOnlyOps, getUniformOpProbs(insertSearchOnlyOps)));
+
+ // Inserts, updates, deletes, and upserts.
+ TestOperation[] insertDeleteUpdateUpsertOps = new TestOperation[] { TestOperation.INSERT, TestOperation.DELETE, TestOperation.UPDATE, TestOperation.UPSERT };
+ workloadConfs.add(new TestWorkloadConf(insertDeleteUpdateUpsertOps, getUniformOpProbs(insertDeleteUpdateUpsertOps)));
+
+ // All operations mixed.
+ TestOperation[] allOps = new TestOperation[] { TestOperation.INSERT, TestOperation.DELETE, TestOperation.UPDATE, TestOperation.UPSERT, TestOperation.POINT_SEARCH, TestOperation.SCAN, TestOperation.DISKORDER_SCAN };
+ workloadConfs.add(new TestWorkloadConf(allOps, getUniformOpProbs(allOps)));
+
+ return workloadConfs;
+ }
+
+ @Override
+ protected int getFileId() {
+ return harness.getBTreeFileId();
+ }
+
+ @Override
+ protected String getIndexTypeName() {
+ return "BTree";
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/multithread/BTreeTestWorker.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/multithread/BTreeTestWorker.java
new file mode 100644
index 0000000..7d8de7d
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/multithread/BTreeTestWorker.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.multithread;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeDuplicateKeyException;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeNonExistentKeyException;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeNotUpdateableException;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.common.AbstractTreeIndexTestWorker;
+import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector;
+import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+public class BTreeTestWorker extends AbstractTreeIndexTestWorker {
+
+ private final BTree btree;
+ private final int numKeyFields;
+ private final ArrayTupleBuilder deleteTb;
+ private final ArrayTupleReference deleteTuple = new ArrayTupleReference();
+
+ public BTreeTestWorker(DataGenThread dataGen, TestOperationSelector opSelector, ITreeIndex index, int numBatches) {
+ super(dataGen, opSelector, index, numBatches);
+ btree = (BTree) index;
+ numKeyFields = btree.getComparatorFactories().length;
+ deleteTb = new ArrayTupleBuilder(numKeyFields);
+ }
+
+ @Override
+ public void performOp(ITupleReference tuple, TestOperation op) throws HyracksDataException, TreeIndexException {
+ BTree.BTreeAccessor accessor = (BTree.BTreeAccessor) indexAccessor;
+ ITreeIndexCursor searchCursor = accessor.createSearchCursor();
+ ITreeIndexCursor diskOrderScanCursor = accessor.createDiskOrderScanCursor();
+ MultiComparator cmp = accessor.getOpContext().cmp;
+ RangePredicate rangePred = new RangePredicate(tuple, tuple, true, true, cmp, cmp);
+
+ switch (op) {
+ case INSERT:
+ try {
+ accessor.insert(tuple);
+ } catch (BTreeDuplicateKeyException e) {
+ // Ignore duplicate keys, since we get random tuples.
+ }
+ break;
+
+ case DELETE:
+ // Create a tuple reference with only key fields.
+ deleteTb.reset();
+ for (int i = 0; i < numKeyFields; i++) {
+ deleteTb.addField(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
+ }
+ deleteTuple.reset(deleteTb.getFieldEndOffsets(), deleteTb.getByteArray());
+ try {
+ accessor.delete(deleteTuple);
+ } catch (BTreeNonExistentKeyException e) {
+ // Ignore non-existant keys, since we get random tuples.
+ }
+ break;
+
+ case UPDATE:
+ try {
+ accessor.update(tuple);
+ } catch (BTreeNonExistentKeyException e) {
+ // Ignore non-existant keys, since we get random tuples.
+ } catch (BTreeNotUpdateableException e) {
+ // Ignore not updateable exception due to numKeys == numFields.
+ }
+ break;
+
+ case UPSERT:
+ accessor.upsert(tuple);
+ // Upsert should not throw. If it does, there's
+ // a bigger problem and the test should fail.
+ break;
+
+ case POINT_SEARCH:
+ searchCursor.reset();
+ rangePred.setLowKey(tuple, true);
+ rangePred.setHighKey(tuple, true);
+ accessor.search(searchCursor, rangePred);
+ consumeCursorTuples(searchCursor);
+ break;
+
+ case SCAN:
+ searchCursor.reset();
+ rangePred.setLowKey(null, true);
+ rangePred.setHighKey(null, true);
+ accessor.search(searchCursor, rangePred);
+ consumeCursorTuples(searchCursor);
+ break;
+
+ case DISKORDER_SCAN:
+ diskOrderScanCursor.reset();
+ accessor.diskOrderScan(diskOrderScanCursor);
+ consumeCursorTuples(diskOrderScanCursor);
+ break;
+
+ default:
+ throw new HyracksDataException("Op " + op.toString() + " not supported.");
+ }
+ }
+
+ private void consumeCursorTuples(ITreeIndexCursor cursor) throws HyracksDataException {
+ try {
+ while(cursor.hasNext()) {
+ cursor.next();
+ }
+ } finally {
+ cursor.close();
+ }
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/multithread/BTreeTestWorkerFactory.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/multithread/BTreeTestWorkerFactory.java
new file mode 100644
index 0000000..dc4d883
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/multithread/BTreeTestWorkerFactory.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.multithread;
+
+import edu.uci.ics.hyracks.storage.am.common.AbstractTreeIndexTestWorker;
+import edu.uci.ics.hyracks.storage.am.common.ITreeIndexTestWorkerFactory;
+import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
+
+public class BTreeTestWorkerFactory implements ITreeIndexTestWorkerFactory {
+ @Override
+ public AbstractTreeIndexTestWorker create(DataGenThread dataGen, TestOperationSelector opSelector,
+ ITreeIndex index, int numBatches) {
+ return new BTreeTestWorker(dataGen, opSelector, index, numBatches);
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/AbstractBTreeTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/AbstractBTreeTest.java
index 9630a1b..f4eca1b 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/AbstractBTreeTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/AbstractBTreeTest.java
@@ -1,76 +1,46 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.btree.util;
-import java.io.File;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.Random;
import java.util.logging.Logger;
import org.junit.After;
import org.junit.Before;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
-import edu.uci.ics.hyracks.test.support.TestUtils;
public abstract class AbstractBTreeTest {
- protected static final Logger LOGGER = Logger.getLogger(AbstractBTreeTest.class.getName());
- public static final long RANDOM_SEED = 50;
-
- private static final int PAGE_SIZE = 256;
- private static final int NUM_PAGES = 10;
- private static final int MAX_OPEN_FILES = 10;
- private static final int HYRACKS_FRAME_SIZE = 128;
-
- protected IHyracksTaskContext ctx;
- protected IBufferCache bufferCache;
- protected int btreeFileId;
-
- protected final Random rnd = new Random();
- protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
- protected final static String tmpDir = System.getProperty("java.io.tmpdir");
- protected final static String sep = System.getProperty("file.separator");
- protected String fileName;
-
- @Before
- public void setUp() throws HyracksDataException {
- fileName = tmpDir + sep + simpleDateFormat.format(new Date());
- ctx = TestUtils.create(getHyracksFrameSize());
- TestStorageManagerComponentHolder.init(getPageSize(), getNumPages(), getMaxOpenFiles());
- bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- btreeFileId = fmp.lookupFileId(file);
- bufferCache.openFile(btreeFileId);
- rnd.setSeed(RANDOM_SEED);
+ protected final Logger LOGGER = Logger.getLogger(BTreeTestHarness.class.getName());
+ protected final BTreeTestHarness harness;
+
+ public AbstractBTreeTest() {
+ harness = new BTreeTestHarness();
}
- @After
+ public AbstractBTreeTest(int pageSize, int numPages, int maxOpenFiles, int hyracksFrameSize) {
+ harness = new BTreeTestHarness(pageSize, numPages, maxOpenFiles, hyracksFrameSize);
+ }
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
public void tearDown() throws HyracksDataException {
- bufferCache.closeFile(btreeFileId);
- bufferCache.close();
- File f = new File(fileName);
- f.deleteOnExit();
- }
-
- public int getPageSize() {
- return PAGE_SIZE;
- }
-
- public int getNumPages() {
- return NUM_PAGES;
- }
-
- public int getHyracksFrameSize() {
- return HYRACKS_FRAME_SIZE;
- }
-
- public int getMaxOpenFiles() {
- return MAX_OPEN_FILES;
+ harness.tearDown();
}
}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestContext.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestContext.java
index f1b03c1..b820f93 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestContext.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestContext.java
@@ -15,49 +15,43 @@
package edu.uci.ics.hyracks.storage.am.btree.util;
-import java.util.TreeSet;
-
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
+import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
@SuppressWarnings("rawtypes")
-public final class BTreeTestContext {
- public final ISerializerDeserializer[] fieldSerdes;
- public final IBufferCache bufferCache;
- public final BTree btree;
- public final IBTreeLeafFrame leafFrame;
- public final IBTreeInteriorFrame interiorFrame;
- public final ITreeIndexMetaDataFrame metaFrame;
- public final ArrayTupleBuilder tupleBuilder;
- public final ArrayTupleReference tuple = new ArrayTupleReference();
- public final TreeSet<CheckTuple> checkTuples = new TreeSet<CheckTuple>();
- public final ITreeIndexAccessor indexAccessor;
-
- public BTreeTestContext(IBufferCache bufferCache, ISerializerDeserializer[] fieldSerdes, BTree btree,
- IBTreeLeafFrame leafFrame, IBTreeInteriorFrame interiorFrame, ITreeIndexMetaDataFrame metaFrame,
- ITreeIndexAccessor indexAccessor) {
- this.bufferCache = bufferCache;
- this.fieldSerdes = fieldSerdes;
- this.btree = btree;
- this.leafFrame = leafFrame;
- this.interiorFrame = interiorFrame;
- this.metaFrame = metaFrame;
- this.indexAccessor = indexAccessor;
- this.tupleBuilder = new ArrayTupleBuilder(fieldSerdes.length);
+public class BTreeTestContext extends OrderedIndexTestContext {
+
+ public BTreeTestContext(ISerializerDeserializer[] fieldSerdes, ITreeIndex treeIndex) {
+ super(fieldSerdes, treeIndex);
}
- public int getFieldCount() {
- return fieldSerdes.length;
- }
-
+ @Override
public int getKeyFieldCount() {
- return btree.getMultiComparator().getKeyFieldCount();
+ BTree btree = (BTree) treeIndex;
+ return btree.getComparatorFactories().length;
}
-}
\ No newline at end of file
+
+ @Override
+ public IBinaryComparatorFactory[] getComparatorFactories() {
+ BTree btree = (BTree) treeIndex;
+ return btree.getComparatorFactories();
+ }
+
+ public static BTreeTestContext create(IBufferCache bufferCache, int btreeFileId, ISerializerDeserializer[] fieldSerdes, int numKeyFields, BTreeLeafFrameType leafType) throws Exception {
+ ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
+ IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
+ BTree btree = BTreeUtils.createBTree(bufferCache, NoOpOperationCallback.INSTANCE, typeTraits, cmpFactories, leafType);
+ btree.create(btreeFileId);
+ btree.open(btreeFileId);
+ BTreeTestContext testCtx = new BTreeTestContext(fieldSerdes, btree);
+ return testCtx;
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestHarness.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestHarness.java
new file mode 100644
index 0000000..1b450d8
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestHarness.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.util;
+
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Random;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.common.api.IOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
+import edu.uci.ics.hyracks.test.support.TestUtils;
+
+public class BTreeTestHarness {
+ public static final BTreeLeafFrameType[] LEAF_FRAMES_TO_TEST = new BTreeLeafFrameType[] {
+ BTreeLeafFrameType.REGULAR_NSM, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM };
+
+ private static final long RANDOM_SEED = 50;
+ private static final int DEFAULT_PAGE_SIZE = 256;
+ private static final int DEFAULT_NUM_PAGES = 100;
+ private static final int DEFAULT_MAX_OPEN_FILES = 10;
+ private static final int DEFAULT_HYRACKS_FRAME_SIZE = 128;
+
+ protected final int pageSize;
+ protected final int numPages;
+ protected final int maxOpenFiles;
+ protected final int hyracksFrameSize;
+
+ protected IHyracksTaskContext ctx;
+ protected IBufferCache bufferCache;
+ protected int btreeFileId;
+
+ protected final Random rnd = new Random();
+ protected final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ protected final String tmpDir = System.getProperty("java.io.tmpdir");
+ protected final String sep = System.getProperty("file.separator");
+ protected String fileName;
+
+ public BTreeTestHarness() {
+ this.pageSize = DEFAULT_PAGE_SIZE;
+ this.numPages = DEFAULT_NUM_PAGES;
+ this.maxOpenFiles = DEFAULT_MAX_OPEN_FILES;
+ this.hyracksFrameSize = DEFAULT_HYRACKS_FRAME_SIZE;
+ }
+
+ public BTreeTestHarness(int pageSize, int numPages, int maxOpenFiles, int hyracksFrameSize) {
+ this.pageSize = pageSize;
+ this.numPages = numPages;
+ this.maxOpenFiles = maxOpenFiles;
+ this.hyracksFrameSize = hyracksFrameSize;
+ }
+
+ public void setUp() throws HyracksDataException {
+ fileName = tmpDir + sep + simpleDateFormat.format(new Date());
+ ctx = TestUtils.create(getHyracksFrameSize());
+ TestStorageManagerComponentHolder.init(pageSize, numPages, maxOpenFiles);
+ bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ FileReference file = new FileReference(new File(fileName));
+ bufferCache.createFile(file);
+ btreeFileId = fmp.lookupFileId(file);
+ bufferCache.openFile(btreeFileId);
+ rnd.setSeed(RANDOM_SEED);
+ }
+
+ public void tearDown() throws HyracksDataException {
+ bufferCache.closeFile(btreeFileId);
+ bufferCache.close();
+ File f = new File(fileName);
+ f.deleteOnExit();
+ }
+
+ public IHyracksTaskContext getHyracksTaskContext() {
+ return ctx;
+ }
+
+ public IBufferCache getBufferCache() {
+ return bufferCache;
+ }
+
+ public int getBTreeFileId() {
+ return btreeFileId;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public Random getRandom() {
+ return rnd;
+ }
+
+ public int getPageSize() {
+ return pageSize;
+ }
+
+ public int getNumPages() {
+ return numPages;
+ }
+
+ public int getHyracksFrameSize() {
+ return hyracksFrameSize;
+ }
+
+ public int getMaxOpenFiles() {
+ return maxOpenFiles;
+ }
+
+ public IOperationCallback getOpCallback() {
+ return NoOpOperationCallback.INSTANCE;
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestUtils.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestUtils.java
deleted file mode 100644
index b5186b0..0000000
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestUtils.java
+++ /dev/null
@@ -1,505 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.btree.util;
-
-import static org.junit.Assert.fail;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.util.Iterator;
-import java.util.NavigableSet;
-import java.util.Random;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
-import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeDuplicateKeyException;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
-import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
-import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-
-@SuppressWarnings("rawtypes")
-public class BTreeTestUtils {
- private static final Logger LOGGER = Logger.getLogger(BTreeTestUtils.class.getName());
-
- public static BTreeTestContext createBTreeTestContext(IBufferCache bufferCache, int btreeFileId, ISerializerDeserializer[] fieldSerdes, int numKeyFields, BTreeLeafFrameType leafType) throws Exception {
- ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes, fieldSerdes.length);
- IBinaryComparator[] cmps = SerdeUtils.serdesToComparators(fieldSerdes, numKeyFields);
-
- BTree btree = BTreeUtils.createBTree(bufferCache, btreeFileId, typeTraits, cmps, leafType);
- btree.create(btreeFileId);
- btree.open(btreeFileId);
- ITreeIndexAccessor indexAccessor = btree.createAccessor();
-
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
- IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) btree.getInteriorFrameFactory().createFrame();
- ITreeIndexMetaDataFrame metaFrame = btree.getFreePageManager().getMetaDataFrameFactory().createFrame();
- BTreeTestContext testCtx = new BTreeTestContext(bufferCache, fieldSerdes, btree, leafFrame, interiorFrame, metaFrame, indexAccessor);
- return testCtx;
- }
-
- private static void compareActualAndExpected(ITupleReference actual, CheckTuple expected, ISerializerDeserializer[] fieldSerdes) throws HyracksDataException {
- for (int i = 0; i < fieldSerdes.length; i++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(
- actual.getFieldData(i), actual.getFieldStart(i),
- actual.getFieldLength(i));
- DataInput dataIn = new DataInputStream(inStream);
- Object actualObj = fieldSerdes[i].deserialize(dataIn);
- if (!actualObj.equals(expected.get(i))) {
- fail("Actual and expected fields do not match.\nExpected: " + expected.get(i) + "\nActual : " + actualObj);
- }
- }
- }
-
- @SuppressWarnings("unchecked")
- private static CheckTuple createCheckTupleFromTuple(ITupleReference tuple, ISerializerDeserializer[] fieldSerdes, int numKeys) throws HyracksDataException {
- CheckTuple checkTuple = new CheckTuple(fieldSerdes.length, numKeys);
- int fieldCount = Math.min(fieldSerdes.length, tuple.getFieldCount());
- for (int i = 0; i < fieldCount; i++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(
- tuple.getFieldData(i), tuple.getFieldStart(i),
- tuple.getFieldLength(i));
- DataInput dataIn = new DataInputStream(inStream);
- Comparable fieldObj = (Comparable)fieldSerdes[i].deserialize(dataIn);
- checkTuple.add(fieldObj);
- }
- return checkTuple;
- }
-
- @SuppressWarnings("unchecked")
- private static void createTupleFromCheckTuple(CheckTuple checkTuple, ArrayTupleBuilder tupleBuilder, ArrayTupleReference tuple, ISerializerDeserializer[] fieldSerdes) throws HyracksDataException {
- int fieldCount = tupleBuilder.getFieldEndOffsets().length;
- DataOutput dos = tupleBuilder.getDataOutput();
- tupleBuilder.reset();
- for (int i = 0; i < fieldCount; i++) {
- fieldSerdes[i].serialize(checkTuple.get(i), dos);
- tupleBuilder.addFieldEndOffset();
- }
- tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
- }
-
- public static void checkOrderedScan(BTreeTestContext testCtx) throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Testing Ordered Scan.");
- }
- ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(testCtx.leafFrame, false);
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- testCtx.indexAccessor.search(scanCursor, nullPred);
- Iterator<CheckTuple> checkIter = testCtx.checkTuples.iterator();
- int actualCount = 0;
- try {
- while (scanCursor.hasNext()) {
- if (!checkIter.hasNext()) {
- fail("Ordered scan returned more answers than expected.\nExpected: " + testCtx.checkTuples.size());
- }
- scanCursor.next();
- CheckTuple expectedTuple = checkIter.next();
- ITupleReference tuple = scanCursor.getTuple();
- compareActualAndExpected(tuple, expectedTuple, testCtx.fieldSerdes);
- actualCount++;
- }
- if (actualCount < testCtx.checkTuples.size()) {
- fail("Ordered scan returned fewer answers than expected.\nExpected: " + testCtx.checkTuples.size() + "\nActual : " + actualCount);
- }
- } finally {
- scanCursor.close();
- }
- }
-
- public static void checkDiskOrderScan(BTreeTestContext testCtx) throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Testing Disk-Order Scan.");
- }
- ITreeIndexCursor diskOrderCursor = new TreeDiskOrderScanCursor(testCtx.leafFrame);
- testCtx.indexAccessor.diskOrderScan(diskOrderCursor);
- int actualCount = 0;
- try {
- while (diskOrderCursor.hasNext()) {
- diskOrderCursor.next();
- ITupleReference tuple = diskOrderCursor.getTuple();
- CheckTuple checkTuple = createCheckTupleFromTuple(tuple, testCtx.fieldSerdes, testCtx.btree.getMultiComparator().getKeyFieldCount());
- if (!testCtx.checkTuples.contains(checkTuple)) {
- fail("Disk-order scan returned unexpected answer: " + checkTuple.toString());
- }
- actualCount++;
- }
- if (actualCount < testCtx.checkTuples.size()) {
- fail("Disk-order scan returned fewer answers than expected.\nExpected: " + testCtx.checkTuples.size() + "\nActual : " + actualCount);
- }
- if (actualCount > testCtx.checkTuples.size()) {
- fail("Disk-order scan returned more answers than expected.\nExpected: " + testCtx.checkTuples.size() + "\nActual : " + actualCount);
- }
- } finally {
- diskOrderCursor.close();
- }
- }
-
- public static void checkRangeSearch(BTreeTestContext testCtx, ITupleReference lowKey, ITupleReference highKey, boolean lowKeyInclusive, boolean highKeyInclusive) throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Testing Range Search.");
- }
- MultiComparator lowKeyCmp = BTreeUtils.getSearchMultiComparator(testCtx.btree.getMultiComparator(), lowKey);
- MultiComparator highKeyCmp = BTreeUtils.getSearchMultiComparator(testCtx.btree.getMultiComparator(), highKey);
- ITreeIndexCursor searchCursor = new BTreeRangeSearchCursor(testCtx.leafFrame, false);
- RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, lowKeyInclusive, highKeyInclusive, lowKeyCmp, highKeyCmp);
- testCtx.indexAccessor.search(searchCursor, rangePred);
- // Get the subset of elements from the expected set within given key range.
- CheckTuple lowKeyCheck = createCheckTupleFromTuple(lowKey, testCtx.fieldSerdes, lowKeyCmp.getKeyFieldCount());
- CheckTuple highKeyCheck = createCheckTupleFromTuple(highKey, testCtx.fieldSerdes, highKeyCmp.getKeyFieldCount());
- NavigableSet<CheckTuple> expectedSubset = null;
- if (lowKeyCmp.getKeyFieldCount() < testCtx.btree.getMultiComparator().getKeyFieldCount() ||
- highKeyCmp.getKeyFieldCount() < testCtx.btree.getMultiComparator().getKeyFieldCount()) {
- // Searching on a key prefix (low key or high key or both).
- expectedSubset = getPrefixExpectedSubset(testCtx.checkTuples, lowKeyCheck, highKeyCheck);
- } else {
- // Searching on all key fields.
- expectedSubset = testCtx.checkTuples.subSet(lowKeyCheck, lowKeyInclusive, highKeyCheck, highKeyInclusive);
- }
- Iterator<CheckTuple> checkIter = expectedSubset.iterator();
- int actualCount = 0;
- try {
- while (searchCursor.hasNext()) {
- if (!checkIter.hasNext()) {
- fail("Range search returned more answers than expected.\nExpected: " + expectedSubset.size());
- }
- searchCursor.next();
- CheckTuple expectedTuple = checkIter.next();
- ITupleReference tuple = searchCursor.getTuple();
- compareActualAndExpected(tuple, expectedTuple, testCtx.fieldSerdes);
- actualCount++;
- }
- if (actualCount < expectedSubset.size()) {
- fail("Range search returned fewer answers than expected.\nExpected: " + expectedSubset.size() + "\nActual : " + actualCount);
- }
- } finally {
- searchCursor.close();
- }
- }
-
- public static void checkPointSearches(BTreeTestContext testCtx) throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Testing Point Searches On All Expected Keys.");
- }
- ITreeIndexCursor searchCursor = new BTreeRangeSearchCursor(testCtx.leafFrame, false);
-
- ArrayTupleBuilder lowKeyBuilder = new ArrayTupleBuilder(testCtx.btree.getMultiComparator().getKeyFieldCount());
- ArrayTupleReference lowKey = new ArrayTupleReference();
- ArrayTupleBuilder highKeyBuilder = new ArrayTupleBuilder(testCtx.btree.getMultiComparator().getKeyFieldCount());
- ArrayTupleReference highKey = new ArrayTupleReference();
- RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, true, true, null, null);
-
- // Iterate through expected tuples, and perform a point search in the BTree to verify the tuple can be reached.
- for (CheckTuple checkTuple : testCtx.checkTuples) {
- createTupleFromCheckTuple(checkTuple, lowKeyBuilder, lowKey, testCtx.fieldSerdes);
- createTupleFromCheckTuple(checkTuple, highKeyBuilder, highKey, testCtx.fieldSerdes);
- MultiComparator lowKeyCmp = BTreeUtils.getSearchMultiComparator(testCtx.btree.getMultiComparator(), lowKey);
- MultiComparator highKeyCmp = BTreeUtils.getSearchMultiComparator(testCtx.btree.getMultiComparator(), highKey);
-
- rangePred.setLowKey(lowKey, true);
- rangePred.setHighKey(highKey, true);
- rangePred.setLowKeyComparator(lowKeyCmp);
- rangePred.setHighKeyComparator(highKeyCmp);
-
- testCtx.indexAccessor.search(searchCursor, rangePred);
-
- try {
- // We expect exactly one answer.
- if (searchCursor.hasNext()) {
- searchCursor.next();
- ITupleReference tuple = searchCursor.getTuple();
- compareActualAndExpected(tuple, checkTuple, testCtx.fieldSerdes);
- }
- if (searchCursor.hasNext()) {
- fail("Point search returned more than one answer.");
- }
- } finally {
- searchCursor.close();
- }
- }
- }
-
- @SuppressWarnings("unchecked")
- // Create a new TreeSet containing the elements satisfying the prefix search.
- // Implementing prefix search by changing compareTo() in CheckTuple does not work.
- public static TreeSet<CheckTuple> getPrefixExpectedSubset(TreeSet<CheckTuple> checkTuples, CheckTuple lowKey, CheckTuple highKey) {
- TreeSet<CheckTuple> expectedSubset = new TreeSet<CheckTuple>();
- Iterator<CheckTuple> iter = checkTuples.iterator();
- while(iter.hasNext()) {
- CheckTuple t = iter.next();
- boolean geLowKey = true;
- boolean leHighKey = true;
- for (int i = 0; i < lowKey.getNumKeys(); i++) {
- if (t.get(i).compareTo(lowKey.get(i)) < 0) {
- geLowKey = false;
- break;
- }
- }
- for (int i = 0; i < highKey.getNumKeys(); i++) {
- if (t.get(i).compareTo(highKey.get(i)) > 0) {
- leHighKey = false;
- break;
- }
- }
- if (geLowKey && leHighKey) {
- expectedSubset.add(t);
- }
- }
- return expectedSubset;
- }
-
- public static void insertIntTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
- int fieldCount = testCtx.getFieldCount();
- int numKeyFields = testCtx.getKeyFieldCount();
- int[] tupleValues = new int[testCtx.getFieldCount()];
- // Scale range of values according to number of keys.
- // For example, for 2 keys we want the square root of numTuples, for 3 keys the cube root of numTuples, etc.
- int maxValue = (int)Math.ceil(Math.pow(numTuples, 1.0/(double)numKeyFields));
- for (int i = 0; i < numTuples; i++) {
- // Set keys.
- for (int j = 0; j < numKeyFields; j++) {
- tupleValues[j] = rnd.nextInt() % maxValue;
- }
- // Set values.
- for (int j = numKeyFields; j < fieldCount; j++) {
- tupleValues[j] = j;
- }
- TupleUtils.createIntegerTuple(testCtx.tupleBuilder, testCtx.tuple, tupleValues);
- if (LOGGER.isLoggable(Level.INFO)) {
- if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
- LOGGER.info("Inserting Tuple " + (i + 1) + "/" + numTuples);
- }
- }
- try {
- testCtx.indexAccessor.insert(testCtx.tuple);
- // Set expected values. Do this only after insertion succeeds because we ignore duplicate keys.
- CheckTuple<Integer> checkTuple = new CheckTuple<Integer>(fieldCount, numKeyFields);
- for(int v : tupleValues) {
- checkTuple.add(v);
- }
- testCtx.checkTuples.add(checkTuple);
- } catch (BTreeDuplicateKeyException e) {
- // Ignore duplicate key insertions.
- }
- }
- }
-
- public static void insertStringTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
- int fieldCount = testCtx.getFieldCount();
- int numKeyFields = testCtx.getKeyFieldCount();
- Object[] tupleValues = new Object[fieldCount];
- for (int i = 0; i < numTuples; i++) {
- if (LOGGER.isLoggable(Level.INFO)) {
- if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
- LOGGER.info("Inserting Tuple " + (i + 1) + "/" + numTuples);
- }
- }
- // Set keys.
- for (int j = 0; j < numKeyFields; j++) {
- int length = (Math.abs(rnd.nextInt()) % 10) + 1;
- tupleValues[j] = getRandomString(length, rnd);
- }
- // Set values.
- for (int j = numKeyFields; j < fieldCount; j++) {
- tupleValues[j] = getRandomString(5, rnd);
- }
- TupleUtils.createTuple(testCtx.tupleBuilder, testCtx.tuple, testCtx.fieldSerdes, tupleValues);
- try {
- testCtx.indexAccessor.insert(testCtx.tuple);
- // Set expected values. Do this only after insertion succeeds because we ignore duplicate keys.
- CheckTuple<String> checkTuple = new CheckTuple<String>(fieldCount, numKeyFields);
- for(Object v : tupleValues) {
- checkTuple.add((String)v);
- }
- testCtx.checkTuples.add(checkTuple);
- } catch (BTreeDuplicateKeyException e) {
- // Ignore duplicate key insertions.
- }
- }
- }
-
- public static void bulkLoadIntTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
- int fieldCount = testCtx.getFieldCount();
- int numKeyFields = testCtx.getKeyFieldCount();
- int[] tupleValues = new int[testCtx.getFieldCount()];
- int maxValue = (int)Math.ceil(Math.pow(numTuples, 1.0/(double)numKeyFields));
- for (int i = 0; i < numTuples; i++) {
- // Set keys.
- for (int j = 0; j < numKeyFields; j++) {
- tupleValues[j] = rnd.nextInt() % maxValue;
- }
- // Set values.
- for (int j = numKeyFields; j < fieldCount; j++) {
- tupleValues[j] = j;
- }
-
- // Set expected values. We also use these as the pre-sorted stream for bulk loading.
- CheckTuple<Integer> checkTuple = new CheckTuple<Integer>(fieldCount, numKeyFields);
- for(int v : tupleValues) {
- checkTuple.add(v);
- }
- testCtx.checkTuples.add(checkTuple);
- }
-
- bulkLoadCheckTuples(testCtx, numTuples);
- }
-
- public static void bulkLoadStringTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
- int fieldCount = testCtx.getFieldCount();
- int numKeyFields = testCtx.getKeyFieldCount();
- String[] tupleValues = new String[fieldCount];
- for (int i = 0; i < numTuples; i++) {
- // Set keys.
- for (int j = 0; j < numKeyFields; j++) {
- int length = (Math.abs(rnd.nextInt()) % 10) + 1;
- tupleValues[j] = getRandomString(length, rnd);
- }
- // Set values.
- for (int j = numKeyFields; j < fieldCount; j++) {
- tupleValues[j] = getRandomString(5, rnd);
- }
- // Set expected values. We also use these as the pre-sorted stream for bulk loading.
- CheckTuple<String> checkTuple = new CheckTuple<String>(fieldCount, numKeyFields);
- for(String v : tupleValues) {
- checkTuple.add(v);
- }
- testCtx.checkTuples.add(checkTuple);
- }
-
- bulkLoadCheckTuples(testCtx, numTuples);
- }
-
- private static void bulkLoadCheckTuples(BTreeTestContext testCtx, int numTuples) throws HyracksDataException, TreeIndexException, PageAllocationException {
- int fieldCount = testCtx.getFieldCount();
- ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
- ArrayTupleReference tuple = new ArrayTupleReference();
- // Perform bulk load.
- IIndexBulkLoadContext bulkLoadCtx = testCtx.btree.beginBulkLoad(0.7f);
- int c = 1;
- for (CheckTuple checkTuple : testCtx.checkTuples) {
- if (LOGGER.isLoggable(Level.INFO)) {
- if (c % (numTuples / 10) == 0) {
- LOGGER.info("Bulk Loading Tuple " + c + "/" + numTuples);
- }
- }
- createTupleFromCheckTuple(checkTuple, tupleBuilder, tuple, testCtx.fieldSerdes);
- testCtx.btree.bulkLoadAddTuple(tuple, bulkLoadCtx);
- c++;
- }
- testCtx.btree.endBulkLoad(bulkLoadCtx);
- }
-
- public static void deleteTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
- ArrayTupleBuilder deleteTupleBuilder = new ArrayTupleBuilder(testCtx.btree.getMultiComparator().getKeyFieldCount());
- ArrayTupleReference deleteTuple = new ArrayTupleReference();
- int numCheckTuples = testCtx.checkTuples.size();
- // Copy CheckTuple references into array, so we can randomly pick from there.
- CheckTuple[] checkTuples = new CheckTuple[numCheckTuples];
- int idx = 0;
- for (CheckTuple checkTuple : testCtx.checkTuples) {
- checkTuples[idx++] = checkTuple;
- }
- for (int i = 0; i < numTuples && numCheckTuples > 0; i++) {
- if (LOGGER.isLoggable(Level.INFO)) {
- if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
- LOGGER.info("Deleting Tuple " + (i + 1) + "/" + numTuples);
- }
- }
- int checkTupleIdx = Math.abs(rnd.nextInt() % numCheckTuples);
- CheckTuple checkTuple = checkTuples[checkTupleIdx];
- createTupleFromCheckTuple(checkTuple, deleteTupleBuilder, deleteTuple, testCtx.fieldSerdes);
- testCtx.indexAccessor.delete(deleteTuple);
-
- // Remove check tuple from expected results.
- testCtx.checkTuples.remove(checkTuple);
-
- // Swap with last "valid" CheckTuple.
- CheckTuple tmp = checkTuples[numCheckTuples - 1];
- checkTuples[numCheckTuples - 1] = checkTuple;
- checkTuples[checkTupleIdx] = tmp;
- numCheckTuples--;
- }
- }
-
- @SuppressWarnings("unchecked")
- public static void updateTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
- int fieldCount = testCtx.btree.getFieldCount();
- int keyFieldCount = testCtx.btree.getMultiComparator().getKeyFieldCount();
- // This is a noop because we can only update non-key fields.
- if (fieldCount == keyFieldCount) {
- return;
- }
- ArrayTupleBuilder updateTupleBuilder = new ArrayTupleBuilder(fieldCount);
- ArrayTupleReference updateTuple = new ArrayTupleReference();
- int numCheckTuples = testCtx.checkTuples.size();
- // Copy CheckTuple references into array, so we can randomly pick from there.
- CheckTuple[] checkTuples = new CheckTuple[numCheckTuples];
- int idx = 0;
- for (CheckTuple checkTuple : testCtx.checkTuples) {
- checkTuples[idx++] = checkTuple;
- }
- for (int i = 0; i < numTuples && numCheckTuples > 0; i++) {
- if (LOGGER.isLoggable(Level.INFO)) {
- if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
- LOGGER.info("Updating Tuple " + (i + 1) + "/" + numTuples);
- }
- }
- int checkTupleIdx = Math.abs(rnd.nextInt() % numCheckTuples);
- CheckTuple checkTuple = checkTuples[checkTupleIdx];
- // Update check tuple's non-key fields.
- for (int j = keyFieldCount; j < fieldCount; j++) {
- Comparable newValue = getRandomUpdateValue(testCtx.fieldSerdes[j], rnd);
- checkTuple.set(j, newValue);
- }
-
- createTupleFromCheckTuple(checkTuple, updateTupleBuilder, updateTuple, testCtx.fieldSerdes);
- testCtx.indexAccessor.update(updateTuple);
-
- // Swap with last "valid" CheckTuple.
- CheckTuple tmp = checkTuples[numCheckTuples - 1];
- checkTuples[numCheckTuples - 1] = checkTuple;
- checkTuples[checkTupleIdx] = tmp;
- numCheckTuples--;
- }
- }
-
- private static Comparable getRandomUpdateValue(ISerializerDeserializer serde, Random rnd) {
- if (serde instanceof IntegerSerializerDeserializer) {
- return Integer.valueOf(rnd.nextInt());
- } else if (serde instanceof UTF8StringSerializerDeserializer) {
- return getRandomString(10, rnd);
- }
- return null;
- }
-
- public static String getRandomString(int length, Random rnd) {
- String s = Long.toHexString(Double.doubleToLongBits(rnd.nextDouble()));
- StringBuilder strBuilder = new StringBuilder();
- for (int i = 0; i < s.length() && i < length; i++) {
- strBuilder.append(s.charAt(Math.abs(rnd.nextInt()) % s.length()));
- }
- return strBuilder.toString();
- }
-}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/pom.xml b/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/pom.xml
index 963903a..5bf44d3 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/pom.xml
+++ b/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/pom.xml
@@ -27,12 +27,6 @@
<dependencies>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
- <artifactId>hyracks-control-nc</artifactId>
- <version>0.2.1-SNAPSHOT</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-invertedindex</artifactId>
<version>0.2.1-SNAPSHOT</version>
<type>jar</type>
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java b/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java
index 7b6bd9f..e086af6 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java
@@ -15,18 +15,15 @@
package edu.uci.ics.hyracks.storage.am.invertedindex;
-import java.io.DataOutput;
import java.io.File;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Random;
import org.junit.After;
import org.junit.Before;
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
@@ -36,26 +33,25 @@
import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexResultCursor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.FixedSizeElementInvertedListBuilder;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
-import edu.uci.ics.hyracks.storage.am.invertedindex.impls.TOccurrenceSearcher;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizer;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.ITokenFactory;
import edu.uci.ics.hyracks.storage.am.invertedindex.util.InvertedIndexUtils;
@@ -86,8 +82,7 @@
// declare btree keys
protected int btreeKeyFieldCount = 1;
- protected IBinaryComparator[] btreeBinCmps = new IBinaryComparator[btreeKeyFieldCount];
- protected MultiComparator btreeCmp = new MultiComparator(btreeBinCmps);
+ protected IBinaryComparatorFactory[] btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
// btree frame factories
protected TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(btreeTypeTraits);
@@ -112,48 +107,38 @@
protected ITypeTraits[] invListTypeTraits = new ITypeTraits[invListFields];
protected int invListKeys = 1;
- protected IBinaryComparator[] invListBinCmps = new IBinaryComparator[invListKeys];
- protected MultiComparator invListCmp = new MultiComparator(invListBinCmps);
+ protected IBinaryComparatorFactory[] invListCmpFactories = new IBinaryComparatorFactory[invListKeys];
protected InvertedIndex invIndex;
protected Random rnd = new Random();
- protected ByteBuffer frame = taskCtx.allocateFrame();
- protected FrameTupleAppender appender = new FrameTupleAppender(taskCtx.getFrameSize());
protected ArrayTupleBuilder tb = new ArrayTupleBuilder(2);
- protected DataOutput dos = tb.getDataOutput();
+ protected ArrayTupleReference tuple = new ArrayTupleReference();
protected ISerializerDeserializer[] insertSerde = { UTF8StringSerializerDeserializer.INSTANCE,
IntegerSerializerDeserializer.INSTANCE };
protected RecordDescriptor insertRecDesc = new RecordDescriptor(insertSerde);
- protected IFrameTupleAccessor accessor = new FrameTupleAccessor(taskCtx.getFrameSize(), insertRecDesc);
-
- protected FrameTupleReference tuple = new FrameTupleReference();
protected ArrayList<ArrayList<Integer>> checkInvLists = new ArrayList<ArrayList<Integer>>();
protected int maxId = 1000000;
- // protected int maxId = 1000;
protected int[] scanCountArray = new int[maxId];
protected ArrayList<Integer> expectedResults = new ArrayList<Integer>();
protected ISerializerDeserializer[] querySerde = { UTF8StringSerializerDeserializer.INSTANCE };
protected RecordDescriptor queryRecDesc = new RecordDescriptor(querySerde);
- protected FrameTupleAppender queryAppender = new FrameTupleAppender(taskCtx.getFrameSize());
protected ArrayTupleBuilder queryTb = new ArrayTupleBuilder(querySerde.length);
- protected DataOutput queryDos = queryTb.getDataOutput();
-
- protected IFrameTupleAccessor queryAccessor = new FrameTupleAccessor(taskCtx.getFrameSize(), queryRecDesc);
- protected FrameTupleReference queryTuple = new FrameTupleReference();
+ protected ArrayTupleReference queryTuple = new ArrayTupleReference();
protected ITokenFactory tokenFactory;
protected IBinaryTokenizer tokenizer;
- protected TOccurrenceSearcher searcher;
- protected IInvertedIndexResultCursor resultCursor;
+ protected IIndexCursor resultCursor;
+ protected abstract void setTokenizer();
+
/**
* Initialize members, generate data, and bulk load the inverted index.
*/
@@ -169,31 +154,31 @@
btreeFileId = fmp.lookupFileId(btreeFile);
bufferCache.openFile(btreeFileId);
- btreeBinCmps[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY).createBinaryComparator();
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
- freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
+ freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
- btree = new BTree(bufferCache, btreeTypeTraits.length, btreeCmp, freePageManager, interiorFrameFactory,
- leafFrameFactory);
+ btree = new BTree(bufferCache, NoOpOperationCallback.INSTANCE, btreeTypeTraits.length, btreeCmpFactories,
+ freePageManager, interiorFrameFactory, leafFrameFactory);
btree.create(btreeFileId);
btree.open(btreeFileId);
// --- INVERTED INDEX ---
+ setTokenizer();
+
bufferCache.createFile(invListsFile);
invListsFileId = fmp.lookupFileId(invListsFile);
bufferCache.openFile(invListsFileId);
invListTypeTraits[0] = IntegerPointable.TYPE_TRAITS;
- invListBinCmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+ invListCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- invIndex = new InvertedIndex(bufferCache, btree, invListTypeTraits, invListCmp);
+ IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
+ invIndex = new InvertedIndex(bufferCache, btree, invListTypeTraits, invListCmpFactories, invListBuilder);
invIndex.open(invListsFileId);
rnd.setSeed(50);
-
- accessor.reset(frame);
- queryAccessor.reset(frame);
}
@After
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java
index 93934ed..9fdc1c4 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java
@@ -29,7 +29,7 @@
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
@@ -51,6 +51,7 @@
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
@@ -58,6 +59,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
@@ -109,10 +111,8 @@
// declare btree keys
int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY).createBinaryComparator();
-
- MultiComparator btreeCmp = new MultiComparator(cmps);
+ IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
+ cmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(btreeTypeTraits);
ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
@@ -121,10 +121,10 @@
ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
- BTree btree = new BTree(bufferCache, btreeTypeTraits.length, btreeCmp, freePageManager, interiorFrameFactory,
- leafFrameFactory);
+ BTree btree = new BTree(bufferCache, NoOpOperationCallback.INSTANCE, btreeTypeTraits.length, cmpFactories,
+ freePageManager, interiorFrameFactory, leafFrameFactory);
btree.create(btreeFileId);
btree.open(btreeFileId);
@@ -133,12 +133,11 @@
invListTypeTraits[0] = IntegerPointable.TYPE_TRAITS;
int invListKeys = 1;
- IBinaryComparator[] invListBinCmps = new IBinaryComparator[invListKeys];
- invListBinCmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+ IBinaryComparatorFactory[] invListCmpFactories = new IBinaryComparatorFactory[invListKeys];
+ invListCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- MultiComparator invListCmp = new MultiComparator(invListBinCmps);
-
- InvertedIndex invIndex = new InvertedIndex(bufferCache, btree, invListTypeTraits, invListCmp);
+ IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
+ InvertedIndex invIndex = new InvertedIndex(bufferCache, btree, invListTypeTraits, invListCmpFactories, invListBuilder);
invIndex.open(invListsFileId);
Random rnd = new Random();
@@ -176,19 +175,14 @@
int addProb = 0;
int addProbStep = 10;
- IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
- InvertedIndex.BulkLoadContext ctx = invIndex.beginBulkLoad(invListBuilder, HYRACKS_FRAME_SIZE,
- BTree.DEFAULT_FILL_FACTOR);
+ IIndexBulkLoadContext ctx = invIndex.beginBulkLoad(BTree.DEFAULT_FILL_FACTOR);
- int totalElements = 0;
for (int i = 0; i < tokens.size(); i++) {
addProb += addProbStep * (i + 1);
for (int j = 0; j < maxId; j++) {
if ((Math.abs(rnd.nextInt()) % addProb) == 0) {
- totalElements++;
-
tb.reset();
UTF8StringSerializerDeserializer.INSTANCE.serialize(tokens.get(i), dos);
tb.addFieldEndOffset();
@@ -203,7 +197,7 @@
tuple.reset(accessor, 0);
try {
- invIndex.bulkLoadAddTuple(ctx, tuple);
+ invIndex.bulkLoadAddTuple(tuple, ctx);
} catch (Exception e) {
e.printStackTrace();
}
@@ -216,7 +210,8 @@
ITreeIndexCursor btreeCursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) leafFrame, false);
FrameTupleReference searchKey = new FrameTupleReference();
- RangePredicate btreePred = new RangePredicate(true, searchKey, searchKey, true, true, btreeCmp, btreeCmp);
+ MultiComparator btreeCmp = MultiComparator.create(cmpFactories);
+ RangePredicate btreePred = new RangePredicate(searchKey, searchKey, true, true, btreeCmp, btreeCmp);
IInvertedListCursor invListCursor = new FixedSizeElementInvertedListCursor(bufferCache, invListsFileId,
invListTypeTraits);
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchPerfTest.java b/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchPerfTest.java
index 1299fcb..33bf0c5 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchPerfTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchPerfTest.java
@@ -28,14 +28,12 @@
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
-import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
-import edu.uci.ics.hyracks.storage.am.invertedindex.impls.FixedSizeElementInvertedListBuilder;
-import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex.InvertedIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndexSearchPredicate;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.OccurrenceThresholdPanicException;
-import edu.uci.ics.hyracks.storage.am.invertedindex.impls.SearchResultCursor;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.TOccurrenceSearcher;
import edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers.ConjunctiveSearchModifier;
import edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers.JaccardSearchModifier;
@@ -55,20 +53,20 @@
protected List<String> tokens = new ArrayList<String>();
- @Before
- public void start() throws Exception {
- super.start();
+ @Override
+ protected void setTokenizer() {
tokenFactory = new UTF8WordTokenFactory();
tokenizer = new DelimitedUTF8StringBinaryTokenizer(true, false,
tokenFactory);
- searcher = new TOccurrenceSearcher(taskCtx, invIndex, tokenizer);
- resultCursor = new SearchResultCursor(
- searcher.createResultFrameTupleAccessor(),
- searcher.createResultTupleReference());
+ }
+
+ @Before
+ public void start() throws Exception {
+ super.start();
loadData();
}
- public void loadData() throws HyracksDataException, TreeIndexException, PageAllocationException {
+ public void loadData() throws HyracksDataException, TreeIndexException {
tokens.add("compilers");
tokens.add("computer");
tokens.add("databases");
@@ -87,37 +85,23 @@
int addProb = 0;
int addProbStep = 10;
- IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(
- invListTypeTraits);
- InvertedIndex.BulkLoadContext ctx = invIndex.beginBulkLoad(
- invListBuilder, HYRACKS_FRAME_SIZE, BTree.DEFAULT_FILL_FACTOR);
+ IIndexBulkLoadContext ctx = invIndex.beginBulkLoad(BTree.DEFAULT_FILL_FACTOR);
- int totalElements = 0;
for (int i = 0; i < tokens.size(); i++) {
addProb += addProbStep * (i + 1);
for (int j = 0; j < maxId; j++) {
if ((Math.abs(rnd.nextInt()) % addProb) == 0) {
-
- totalElements++;
-
tb.reset();
UTF8StringSerializerDeserializer.INSTANCE.serialize(
- tokens.get(i), dos);
+ tokens.get(i), tb.getDataOutput());
tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(j, dos);
+ IntegerSerializerDeserializer.INSTANCE.serialize(j, tb.getDataOutput());
tb.addFieldEndOffset();
-
+ tuple.reset(tb.getFieldEndOffsets(), tb.getByteArray());
checkInvLists.get(i).add(j);
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(),
- 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
try {
- invIndex.bulkLoadAddTuple(ctx, tuple);
+ invIndex.bulkLoadAddTuple(tuple, ctx);
} catch (Exception e) {
e.printStackTrace();
}
@@ -170,6 +154,9 @@
rnd.setSeed(50);
+ InvertedIndexAccessor accessor = (InvertedIndexAccessor) invIndex.createAccessor();
+ InvertedIndexSearchPredicate searchPred = new InvertedIndexSearchPredicate(tokenizer, searchModifier);
+
// generate random queries
int[] queryTokenIndexes = new int[tokens.size()];
for (int i = 0; i < numQueries; i++) {
@@ -182,31 +169,34 @@
StringBuilder strBuilder = new StringBuilder();
for (int j = 0; j < numQueryTokens; j++) {
strBuilder.append(tokens.get(queryTokenIndexes[j]));
- if (j + 1 != numQueryTokens)
+ if (j + 1 != numQueryTokens) {
strBuilder.append(" ");
+ }
}
String queryString = strBuilder.toString();
+ // Serialize query.
queryTb.reset();
UTF8StringSerializerDeserializer.INSTANCE.serialize(queryString,
- queryDos);
+ queryTb.getDataOutput());
queryTb.addFieldEndOffset();
+ queryTuple.reset(queryTb.getFieldEndOffsets(), queryTb.getByteArray());
- queryAppender.reset(frame, true);
- queryAppender.append(queryTb.getFieldEndOffsets(),
- queryTb.getByteArray(), 0, queryTb.getSize());
- queryTuple.reset(queryAccessor, 0);
-
+ // Set query tuple in search predicate.
+ searchPred.setQueryTuple(queryTuple);
+ searchPred.setQueryFieldIndex(0);
+
boolean panic = false;
+ resultCursor = accessor.createSearchCursor();
int repeats = 1;
double totalTime = 0;
for (int j = 0; j < repeats; j++) {
long timeStart = System.currentTimeMillis();
try {
- searcher.reset();
- searcher.search(resultCursor, queryTuple, 0, searchModifier);
+ resultCursor.reset();
+ accessor.search(resultCursor, searchPred);
} catch (OccurrenceThresholdPanicException e) {
panic = true;
}
@@ -220,10 +210,9 @@
}
if (!panic) {
-
+ TOccurrenceSearcher searcher = (TOccurrenceSearcher) accessor.getSearcher();
fillExpectedResults(queryTokenIndexes, numQueryTokens,
searcher.getOccurrenceThreshold());
-
// verify results
int checkIndex = 0;
while (resultCursor.hasNext()) {
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchTest.java b/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchTest.java
index eb2b39c..c3c9b99 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchTest.java
@@ -25,20 +25,18 @@
import org.junit.Before;
import org.junit.Test;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ByteArrayAccessibleOutputStream;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.data.std.util.ByteArrayAccessibleOutputStream;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
-import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
-import edu.uci.ics.hyracks.storage.am.invertedindex.impls.FixedSizeElementInvertedListBuilder;
-import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex.InvertedIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndexSearchPredicate;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.OccurrenceThresholdPanicException;
-import edu.uci.ics.hyracks.storage.am.invertedindex.impls.SearchResultCursor;
-import edu.uci.ics.hyracks.storage.am.invertedindex.impls.TOccurrenceSearcher;
import edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers.ConjunctiveSearchModifier;
import edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers.EditDistanceSearchModifier;
import edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers.JaccardSearchModifier;
@@ -52,16 +50,22 @@
protected List<String> firstNames = new ArrayList<String>();
protected List<String> lastNames = new ArrayList<String>();
- @Before
- public void start() throws Exception {
- super.start();
+ protected IBinaryComparator[] btreeBinCmps;
+
+ @Override
+ protected void setTokenizer() {
tokenFactory = new UTF8NGramTokenFactory();
tokenizer = new NGramUTF8StringBinaryTokenizer(3, false, true, false,
tokenFactory);
- searcher = new TOccurrenceSearcher(taskCtx, invIndex, tokenizer);
- resultCursor = new SearchResultCursor(
- searcher.createResultFrameTupleAccessor(),
- searcher.createResultTupleReference());
+ }
+
+ @Before
+ public void start() throws Exception {
+ super.start();
+ btreeBinCmps = new IBinaryComparator[btreeCmpFactories.length];
+ for (int i = 0; i < btreeCmpFactories.length; i++) {
+ btreeBinCmps[i] = btreeCmpFactories[i].createBinaryComparator();
+ }
generateDataStrings();
loadData();
}
@@ -130,7 +134,7 @@
}
}
- public void loadData() throws IOException, TreeIndexException, PageAllocationException {
+ public void loadData() throws IOException, TreeIndexException {
List<TokenIdPair> pairs = new ArrayList<TokenIdPair>();
// generate pairs for subsequent sorting and bulk-loading
int id = 0;
@@ -139,38 +143,28 @@
DataOutputStream dos = new DataOutputStream(baaos);
UTF8StringSerializerDeserializer.INSTANCE.serialize(s, dos);
tokenizer.reset(baaos.getByteArray(), 0, baaos.size());
- int tokenCount = 0;
while (tokenizer.hasNext()) {
tokenizer.next();
IToken token = tokenizer.getToken();
pairs.add(new TokenIdPair(token, id));
- ++tokenCount;
}
++id;
}
Collections.sort(pairs);
// bulk load index
- IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(
- invListTypeTraits);
- InvertedIndex.BulkLoadContext ctx = invIndex.beginBulkLoad(
- invListBuilder, HYRACKS_FRAME_SIZE, BTree.DEFAULT_FILL_FACTOR);
+ IIndexBulkLoadContext ctx = invIndex.beginBulkLoad(BTree.DEFAULT_FILL_FACTOR);
for (TokenIdPair t : pairs) {
tb.reset();
tb.addField(t.baaos.getByteArray(), 0,
t.baaos.getByteArray().length);
- IntegerSerializerDeserializer.INSTANCE.serialize(t.id, dos);
+ IntegerSerializerDeserializer.INSTANCE.serialize(t.id, tb.getDataOutput());
tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0,
- tb.getSize());
-
- tuple.reset(accessor, 0);
+ tuple.reset(tb.getFieldEndOffsets(), tb.getByteArray());
try {
- invIndex.bulkLoadAddTuple(ctx, tuple);
+ invIndex.bulkLoadAddTuple(tuple, ctx);
} catch (Exception e) {
e.printStackTrace();
}
@@ -188,28 +182,34 @@
rnd.setSeed(50);
+ InvertedIndexAccessor accessor = (InvertedIndexAccessor) invIndex.createAccessor();
+ InvertedIndexSearchPredicate searchPred = new InvertedIndexSearchPredicate(tokenizer, searchModifier);
+
for (int i = 0; i < numQueries; i++) {
int queryIndex = Math.abs(rnd.nextInt() % dataStrings.size());
String queryString = dataStrings.get(queryIndex);
+ // Serialize query.
queryTb.reset();
UTF8StringSerializerDeserializer.INSTANCE.serialize(queryString,
- queryDos);
+ queryTb.getDataOutput());
queryTb.addFieldEndOffset();
+ queryTuple.reset(queryTb.getFieldEndOffsets(), queryTb.getByteArray());
- queryAppender.reset(frame, true);
- queryAppender.append(queryTb.getFieldEndOffsets(),
- queryTb.getByteArray(), 0, queryTb.getSize());
- queryTuple.reset(queryAccessor, 0);
-
+ // Set query tuple in search predicate.
+ searchPred.setQueryTuple(queryTuple);
+ searchPred.setQueryFieldIndex(0);
+
+ resultCursor = accessor.createSearchCursor();
+
int repeats = 1;
double totalTime = 0;
for (int j = 0; j < repeats; j++) {
long timeStart = System.currentTimeMillis();
try {
- searcher.reset();
- searcher.search(resultCursor, queryTuple, 0, searchModifier);
+ resultCursor.reset();
+ accessor.search(resultCursor, searchPred);
} catch (OccurrenceThresholdPanicException e) {
// ignore panic queries
}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/pom.xml b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/pom.xml
index 5800867..e071098 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/pom.xml
+++ b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/pom.xml
@@ -33,12 +33,6 @@
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
- <artifactId>hyracks-control-nc</artifactId>
- <version>0.2.1-SNAPSHOT</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-rtree</artifactId>
<version>0.2.1-SNAPSHOT</version>
<type>jar</type>
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeTest.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeTest.java
deleted file mode 100644
index 8f619a4..0000000
--- a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeTest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.rtree;
-
-import java.io.File;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.logging.Logger;
-
-import org.junit.AfterClass;
-
-public abstract class AbstractRTreeTest {
-
- protected static final Logger LOGGER = Logger.getLogger(AbstractRTreeTest.class.getName());
- protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat(
- "ddMMyy-hhmmssSS");
- protected final static String tmpDir = System.getProperty("java.io.tmpdir");
- protected final static String sep = System.getProperty("file.separator");
- protected final static String fileName = tmpDir + sep
- + simpleDateFormat.format(new Date());
-
- @AfterClass
- public static void cleanup() throws Exception {
- File f = new File(fileName);
- f.deleteOnExit();
- }
-}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeBulkLoadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeBulkLoadTest.java
new file mode 100644
index 0000000..58bca10
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeBulkLoadTest.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeBulkLoadTest;
+import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.rtree.utils.RTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.rtree.utils.RTreeTestHarness;
+
+@SuppressWarnings("rawtypes")
+public class RTreeBulkLoadTest extends AbstractRTreeBulkLoadTest {
+
+ public RTreeBulkLoadTest() {
+ super(1);
+ }
+
+ private final RTreeTestHarness harness = new RTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
+ IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys) throws Exception {
+ return RTreeTestContext.create(harness.getBufferCache(), harness.getTreeFileId(), fieldSerdes,
+ valueProviderFactories, numKeys);
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeDeleteTest.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeDeleteTest.java
new file mode 100644
index 0000000..42e933e
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeDeleteTest.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeDeleteTest;
+import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.rtree.utils.RTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.rtree.utils.RTreeTestHarness;
+
+@SuppressWarnings("rawtypes")
+public class RTreeDeleteTest extends AbstractRTreeDeleteTest {
+
+ private final RTreeTestHarness harness = new RTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
+ IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys) throws Exception {
+ return RTreeTestContext.create(harness.getBufferCache(), harness.getTreeFileId(), fieldSerdes,
+ valueProviderFactories, numKeys);
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeExamplesTest.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeExamplesTest.java
new file mode 100644
index 0000000..c72338e
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeExamplesTest.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeExamplesTest;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
+import edu.uci.ics.hyracks.storage.am.rtree.utils.RTreeTestHarness;
+
+public class RTreeExamplesTest extends AbstractRTreeExamplesTest {
+ private final RTreeTestHarness harness = new RTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories)
+ throws TreeIndexException {
+ return RTreeUtils.createRTree(harness.getBufferCache(), typeTraits,
+ valueProviderFactories, rtreeCmpFactories);
+ }
+
+ @Override
+ protected int getIndexFileId() {
+ return harness.getTreeFileId();
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeInsertTest.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeInsertTest.java
new file mode 100644
index 0000000..6efa620
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeInsertTest.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeInsertTest;
+import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.rtree.utils.RTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.rtree.utils.RTreeTestHarness;
+
+/**
+ * Tests the BTree insert operation with strings and integer fields using
+ * various numbers of key and payload fields.
+ *
+ * Each tests first fills a BTree with randomly generated tuples. We compare the
+ * following operations against expected results: 1. Point searches for all
+ * tuples. 2. Ordered scan. 3. Disk-order scan. 4. Range search (and prefix
+ * search for composite keys).
+ *
+ */
+@SuppressWarnings("rawtypes")
+public class RTreeInsertTest extends AbstractRTreeInsertTest {
+
+ private final RTreeTestHarness harness = new RTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
+ IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys) throws Exception {
+ return RTreeTestContext.create(harness.getBufferCache(), harness.getTreeFileId(), fieldSerdes,
+ valueProviderFactories, numKeys);
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeSearchCursorTest.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
new file mode 100644
index 0000000..8332c11
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree;
+
+import java.util.ArrayList;
+import java.util.Random;
+import java.util.logging.Level;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.rtree.RTreeCheckTuple;
+import edu.uci.ics.hyracks.storage.am.rtree.RTreeTestUtils;
+import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.SearchPredicate;
+import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
+import edu.uci.ics.hyracks.storage.am.rtree.utils.AbstractRTreeTest;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+public class RTreeSearchCursorTest extends AbstractRTreeTest {
+
+ private final RTreeTestUtils rTreeTestUtils;
+ private Random rnd = new Random(50);
+
+ public RTreeSearchCursorTest() {
+ this.rTreeTestUtils = new RTreeTestUtils();
+ }
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ super.setUp();
+ }
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ @Test
+ public void rangeSearchTest() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("TESTING RANGE SEARCH CURSOR FOR RTREE");
+ }
+
+ IBufferCache bufferCache = harness.getBufferCache();
+ int rtreeFileId = harness.getTreeFileId();
+
+ // Declare fields.
+ int fieldCount = 5;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[3] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[4] = IntegerPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+
+ // Declare keys.
+ int keyFieldCount = 4;
+ IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
+ cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ cmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ cmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ cmpFactories.length, IntegerPointable.FACTORY);
+
+ RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+
+ ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory,
+ valueProviderFactories);
+
+ IRTreeInteriorFrame interiorFrame = (IRTreeInteriorFrame) interiorFrameFactory.createFrame();
+ IRTreeLeafFrame leafFrame = (IRTreeLeafFrame) leafFrameFactory.createFrame();
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
+
+ RTree rtree = new RTree(bufferCache, fieldCount, cmpFactories, freePageManager, interiorFrameFactory,
+ leafFrameFactory);
+ rtree.create(rtreeFileId);
+ rtree.open(rtreeFileId);
+
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ ITreeIndexAccessor indexAccessor = rtree.createAccessor();
+ int numInserts = 10000;
+ ArrayList<RTreeCheckTuple> checkTuples = new ArrayList<RTreeCheckTuple>();
+ for (int i = 0; i < numInserts; i++) {
+ int p1x = rnd.nextInt();
+ int p1y = rnd.nextInt();
+ int p2x = rnd.nextInt();
+ int p2y = rnd.nextInt();
+
+ int pk = rnd.nextInt();;
+
+ TupleUtils.createIntegerTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x),
+ Math.max(p1y, p2y), pk);
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ RTreeCheckTuple checkTuple = new RTreeCheckTuple(fieldCount, keyFieldCount);
+ checkTuple.add(Math.min(p1x, p2x));
+ checkTuple.add(Math.min(p1y, p2y));
+ checkTuple.add(Math.max(p1x, p2x));
+ checkTuple.add(Math.max(p1y, p2y));
+ checkTuple.add(pk);
+
+ checkTuples.add(checkTuple);
+ }
+
+ // Build key.
+ ArrayTupleBuilder keyTb = new ArrayTupleBuilder(keyFieldCount);
+ ArrayTupleReference key = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(keyTb, key, -1000, -1000, 1000, 1000);
+
+ MultiComparator cmp = MultiComparator.create(cmpFactories);
+ ITreeIndexCursor searchCursor = new RTreeSearchCursor(interiorFrame, leafFrame);
+ SearchPredicate searchPredicate = new SearchPredicate(key, cmp);
+
+ RTreeCheckTuple keyCheck = (RTreeCheckTuple) rTreeTestUtils.createCheckTupleFromTuple(key, fieldSerdes,
+ keyFieldCount);
+ ArrayList<RTreeCheckTuple> expectedResult = rTreeTestUtils.getRangeSearchExpectedResults(checkTuples, keyCheck);
+
+ rTreeTestUtils.getRangeSearchExpectedResults(checkTuples, keyCheck);
+ indexAccessor.search(searchCursor, searchPredicate);
+
+ rTreeTestUtils.checkExpectedResults(searchCursor, expectedResult, fieldSerdes, keyFieldCount, null);
+
+ rtree.close();
+ }
+
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTest.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTest.java
deleted file mode 100644
index 9fe2c94..0000000
--- a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTest.java
+++ /dev/null
@@ -1,769 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.rtree;
-
-import java.io.DataOutput;
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.util.Random;
-import java.util.logging.Level;
-
-import org.junit.Test;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
-import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
-import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexStats;
-import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexStatsGatherer;
-import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeFrame;
-import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
-import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
-import edu.uci.ics.hyracks.test.support.TestUtils;
-
-public class RTreeTest extends AbstractRTreeTest {
- private static final int PAGE_SIZE = 256;
- private static final int NUM_PAGES = 10;
- private static final int MAX_OPEN_FILES = 10;
- private static final int HYRACKS_FRAME_SIZE = 128;
- private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
-
- // create an R-tree of two dimensions
- // fill the R-tree with random values using insertions
- // perform ordered scan
- @Test
- public void test01() throws Exception {
-
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- // declare keys
- int keyFieldCount = 4;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
- cmps[1] = cmps[0];
- cmps[2] = cmps[0];
- cmps[3] = cmps[0];
-
- // declare tuple fields
- int fieldCount = 7;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = DoublePointable.TYPE_TRAITS;
- typeTraits[1] = DoublePointable.TYPE_TRAITS;
- typeTraits[2] = DoublePointable.TYPE_TRAITS;
- typeTraits[3] = DoublePointable.TYPE_TRAITS;
- typeTraits[4] = DoublePointable.TYPE_TRAITS;
- typeTraits[5] = DoublePointable.TYPE_TRAITS;
- typeTraits[6] = DoublePointable.TYPE_TRAITS;
-
- // create value providers
- IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
- cmps.length, DoublePointable.FACTORY);
-
- MultiComparator cmp = new MultiComparator(cmps);
-
- RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
-
- ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory,
- valueProviderFactories);
- ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory,
- valueProviderFactories);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
-
- IRTreeFrame interiorFrame = (IRTreeFrame) interiorFrameFactory.createFrame();
- IRTreeFrame leafFrame = (IRTreeFrame) leafFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
-
- RTree rtree = new RTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
- rtree.create(fileId);
- rtree.open(fileId);
-
- ByteBuffer hyracksFrame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- DataOutput dos = tb.getDataOutput();
-
- @SuppressWarnings("rawtypes")
- ISerializerDeserializer[] recDescSers = { DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(hyracksFrame);
- FrameTupleReference tuple = new FrameTupleReference();
-
- ITreeIndexAccessor indexAccessor = rtree.createAccessor();
-
- Random rnd = new Random();
- rnd.setSeed(50);
-
- Random rnd2 = new Random();
- rnd2.setSeed(50);
- for (int i = 0; i < 5000; i++) {
-
- double p1x = rnd.nextDouble();
- double p1y = rnd.nextDouble();
- double p2x = rnd.nextDouble();
- double p2y = rnd.nextDouble();
-
- double pk1 = rnd2.nextDouble();
- int pk2 = rnd2.nextInt();
- double pk3 = rnd2.nextDouble();
-
- tb.reset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
- tb.addFieldEndOffset();
-
- appender.reset(hyracksFrame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("INSERTING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
- + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
- }
- }
-
- try {
- indexAccessor.insert(tuple);
- } catch (TreeIndexException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- String rtreeStats = rtree.printStats();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(rtreeStats);
- }
-
- // disk-order scan
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("DISK-ORDER SCAN:");
- }
- TreeDiskOrderScanCursor diskOrderCursor = new TreeDiskOrderScanCursor(leafFrame);
- indexAccessor.diskOrderScan(diskOrderCursor);
- try {
- while (diskOrderCursor.hasNext()) {
- diskOrderCursor.next();
- ITupleReference frameTuple = diskOrderCursor.getTuple();
- String rec = TupleUtils.printTuple(frameTuple, recDescSers);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(rec);
- }
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- diskOrderCursor.close();
- }
-
- TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId,
- rtree.getRootPageId());
- TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame);
- String string = stats.toString();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(string);
- }
-
- rtree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
-
- }
-
- // create an R-tree of two dimensions
- // fill the R-tree with random values using insertions
- // and then delete all the tuples which result of an empty R-tree
- @Test
- public void test02() throws Exception {
-
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- // declare keys
- int keyFieldCount = 4;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
- cmps[1] = cmps[0];
- cmps[2] = cmps[0];
- cmps[3] = cmps[0];
-
- // declare tuple fields
- int fieldCount = 7;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = DoublePointable.TYPE_TRAITS;
- typeTraits[1] = DoublePointable.TYPE_TRAITS;
- typeTraits[2] = DoublePointable.TYPE_TRAITS;
- typeTraits[3] = DoublePointable.TYPE_TRAITS;
- typeTraits[4] = DoublePointable.TYPE_TRAITS;
- typeTraits[5] = DoublePointable.TYPE_TRAITS;
- typeTraits[6] = DoublePointable.TYPE_TRAITS;
-
- // create value providers
- IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
- cmps.length, DoublePointable.FACTORY);
-
- MultiComparator cmp = new MultiComparator(cmps);
-
- RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
-
- ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory,
- valueProviderFactories);
- ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory,
- valueProviderFactories);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
-
- IRTreeFrame interiorFrame = (IRTreeFrame) interiorFrameFactory.createFrame();
- IRTreeFrame leafFrame = (IRTreeFrame) leafFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
-
- RTree rtree = new RTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
- rtree.create(fileId);
- rtree.open(fileId);
-
- ByteBuffer hyracksFrame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- DataOutput dos = tb.getDataOutput();
-
- @SuppressWarnings("rawtypes")
- ISerializerDeserializer[] recDescSers = { DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(hyracksFrame);
- FrameTupleReference tuple = new FrameTupleReference();
-
- ITreeIndexAccessor indexAccessor = rtree.createAccessor();
-
- Random rnd = new Random();
- rnd.setSeed(50);
-
- for (int i = 0; i < 5000; i++) {
-
- double p1x = rnd.nextDouble();
- double p1y = rnd.nextDouble();
- double p2x = rnd.nextDouble();
- double p2y = rnd.nextDouble();
-
- double pk1 = rnd.nextDouble();
- int pk2 = rnd.nextInt();
- double pk3 = rnd.nextDouble();
-
- tb.reset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
- tb.addFieldEndOffset();
-
- appender.reset(hyracksFrame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("INSERTING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
- + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
- }
- }
-
- try {
- indexAccessor.insert(tuple);
- } catch (TreeIndexException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- String rtreeStats = rtree.printStats();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(rtreeStats);
- }
-
- rnd.setSeed(50);
- for (int i = 0; i < 5000; i++) {
-
- double p1x = rnd.nextDouble();
- double p1y = rnd.nextDouble();
- double p2x = rnd.nextDouble();
- double p2y = rnd.nextDouble();
-
- double pk1 = rnd.nextDouble();
- int pk2 = rnd.nextInt();
- double pk3 = rnd.nextDouble();
-
- tb.reset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
- tb.addFieldEndOffset();
-
- appender.reset(hyracksFrame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("DELETING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
- + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
- }
- }
-
- try {
- indexAccessor.delete(tuple);
- } catch (TreeIndexException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId,
- rtree.getRootPageId());
- TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame);
- String string = stats.toString();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(string);
- }
-
- rtree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
-
- }
-
- // create an R-tree of three dimensions
- // fill the R-tree with random values using insertions
- // perform ordered scan
- @Test
- public void test03() throws Exception {
-
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- // declare keys
- int keyFieldCount = 6;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
- cmps[1] = cmps[0];
- cmps[2] = cmps[0];
- cmps[3] = cmps[0];
- cmps[4] = cmps[0];
- cmps[5] = cmps[0];
-
- // declare tuple fields
- int fieldCount = 9;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = DoublePointable.TYPE_TRAITS;
- typeTraits[1] = DoublePointable.TYPE_TRAITS;
- typeTraits[2] = DoublePointable.TYPE_TRAITS;
- typeTraits[3] = DoublePointable.TYPE_TRAITS;
- typeTraits[4] = DoublePointable.TYPE_TRAITS;
- typeTraits[5] = DoublePointable.TYPE_TRAITS;
- typeTraits[6] = DoublePointable.TYPE_TRAITS;
- typeTraits[7] = IntegerPointable.TYPE_TRAITS;
- typeTraits[8] = DoublePointable.TYPE_TRAITS;
-
- // create value providers
- IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
- cmps.length, DoublePointable.FACTORY);
-
- MultiComparator cmp = new MultiComparator(cmps);
-
- RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
-
- ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory,
- valueProviderFactories);
- ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory,
- valueProviderFactories);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
-
- IRTreeFrame interiorFrame = (IRTreeFrame) interiorFrameFactory.createFrame();
- IRTreeFrame leafFrame = (IRTreeFrame) leafFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
-
- RTree rtree = new RTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
- rtree.create(fileId);
- rtree.open(fileId);
-
- ByteBuffer hyracksFrame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- DataOutput dos = tb.getDataOutput();
-
- @SuppressWarnings("rawtypes")
- ISerializerDeserializer[] recDescSers = { DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(hyracksFrame);
- FrameTupleReference tuple = new FrameTupleReference();
-
- ITreeIndexAccessor indexAccessor = rtree.createAccessor();
-
- Random rnd = new Random();
- rnd.setSeed(50);
-
- for (int i = 0; i < 5000; i++) {
-
- double p1x = rnd.nextDouble();
- double p1y = rnd.nextDouble();
- double p1z = rnd.nextDouble();
- double p2x = rnd.nextDouble();
- double p2y = rnd.nextDouble();
- double p2z = rnd.nextDouble();
-
- double pk1 = rnd.nextDouble();
- int pk2 = rnd.nextInt();
- double pk3 = rnd.nextDouble();
-
- tb.reset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1z, p2z), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1z, p2z), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
- tb.addFieldEndOffset();
-
- appender.reset(hyracksFrame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("INSERTING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
- + Math.min(p1z, p2z) + " " + " " + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y) + " "
- + Math.max(p1z, p2z));
- }
- }
-
- try {
- indexAccessor.insert(tuple);
- } catch (TreeIndexException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- String rtreeStats = rtree.printStats();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(rtreeStats);
- }
-
- // disk-order scan
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("DISK-ORDER SCAN:");
- }
- TreeDiskOrderScanCursor diskOrderCursor = new TreeDiskOrderScanCursor(leafFrame);
- indexAccessor.diskOrderScan(diskOrderCursor);
- try {
- while (diskOrderCursor.hasNext()) {
- diskOrderCursor.next();
- ITupleReference frameTuple = diskOrderCursor.getTuple();
- String rec = TupleUtils.printTuple(frameTuple, recDescSers);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(rec);
- }
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- diskOrderCursor.close();
- }
-
- TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId,
- rtree.getRootPageId());
- TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame);
- String string = stats.toString();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(string);
- }
-
- rtree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
-
- }
-
- // create an R-tree of two dimensions
- // fill the R-tree with random integer key values using insertions
- // perform ordered scan
- @Test
- public void test04() throws Exception {
-
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- // declare keys
- int keyFieldCount = 4;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
- cmps[1] = cmps[0];
- cmps[2] = cmps[0];
- cmps[3] = cmps[0];
-
- // declare tuple fields
- int fieldCount = 7;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = IntegerPointable.TYPE_TRAITS;
- typeTraits[1] = IntegerPointable.TYPE_TRAITS;
- typeTraits[2] = IntegerPointable.TYPE_TRAITS;
- typeTraits[3] = IntegerPointable.TYPE_TRAITS;
- typeTraits[4] = DoublePointable.TYPE_TRAITS;
- typeTraits[5] = IntegerPointable.TYPE_TRAITS;
- typeTraits[6] = DoublePointable.TYPE_TRAITS;
-
- // create value providers
- IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
- cmps.length, IntegerPointable.FACTORY);
-
- MultiComparator cmp = new MultiComparator(cmps);
-
- RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
-
- ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory,
- valueProviderFactories);
- ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory,
- valueProviderFactories);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
-
- IRTreeFrame interiorFrame = (IRTreeFrame) interiorFrameFactory.createFrame();
- IRTreeFrame leafFrame = (IRTreeFrame) leafFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
-
- RTree rtree = new RTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
- rtree.create(fileId);
- rtree.open(fileId);
-
- ByteBuffer hyracksFrame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- DataOutput dos = tb.getDataOutput();
-
- @SuppressWarnings("rawtypes")
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(hyracksFrame);
- FrameTupleReference tuple = new FrameTupleReference();
-
- ITreeIndexAccessor indexAccessor = rtree.createAccessor();
-
- Random rnd = new Random();
- rnd.setSeed(50);
-
- Random rnd2 = new Random();
- rnd2.setSeed(50);
- for (int i = 0; i < 5000; i++) {
-
- int p1x = rnd.nextInt();
- int p1y = rnd.nextInt();
- int p2x = rnd.nextInt();
- int p2y = rnd.nextInt();
-
- double pk1 = rnd2.nextDouble();
- int pk2 = rnd2.nextInt();
- double pk3 = rnd2.nextDouble();
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
- tb.addFieldEndOffset();
-
- appender.reset(hyracksFrame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("INSERTING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
- + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
- }
- }
-
- try {
- indexAccessor.insert(tuple);
- } catch (TreeIndexException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- String rtreeStats = rtree.printStats();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(rtreeStats);
- }
-
- // disk-order scan
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("DISK-ORDER SCAN:");
- }
- TreeDiskOrderScanCursor diskOrderCursor = new TreeDiskOrderScanCursor(leafFrame);
- indexAccessor.diskOrderScan(diskOrderCursor);
- try {
- while (diskOrderCursor.hasNext()) {
- diskOrderCursor.next();
- ITupleReference frameTuple = diskOrderCursor.getTuple();
- String rec = TupleUtils.printTuple(frameTuple, recDescSers);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(rec);
- }
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- diskOrderCursor.close();
- }
-
- TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId,
- rtree.getRootPageId());
- TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame);
- String string = stats.toString();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(string);
- }
-
- rtree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
-
- }
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/SearchCursorTest.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/SearchCursorTest.java
deleted file mode 100644
index a232e37..0000000
--- a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/SearchCursorTest.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.rtree;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Random;
-import java.util.logging.Level;
-
-import org.junit.Test;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
-import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
-import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
-import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeSearchCursor;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.SearchPredicate;
-import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
-import edu.uci.ics.hyracks.test.support.TestUtils;
-
-public class SearchCursorTest extends AbstractRTreeTest {
- private static final int PAGE_SIZE = 256;
- private static final int NUM_PAGES = 10;
- private static final int MAX_OPEN_FILES = 10;
- private static final int HYRACKS_FRAME_SIZE = 128;
- private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
-
- // create an R-tree of two dimensions
- // fill the R-tree with random values using insertions
- // and then perform range search
- @Test
- public void searchCursorTest() throws Exception {
-
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- // declare keys
- int keyFieldCount = 4;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
- cmps[1] = cmps[0];
- cmps[2] = cmps[0];
- cmps[3] = cmps[0];
-
- // declare tuple fields
- int fieldCount = 5;
- ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- typeTraits[0] = DoublePointable.TYPE_TRAITS;
- typeTraits[1] = DoublePointable.TYPE_TRAITS;
- typeTraits[2] = DoublePointable.TYPE_TRAITS;
- typeTraits[3] = DoublePointable.TYPE_TRAITS;
- typeTraits[4] = IntegerPointable.TYPE_TRAITS;
-
- // create value providers
- IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
- cmps.length, DoublePointable.FACTORY);
-
- MultiComparator cmp = new MultiComparator(cmps);
-
- RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
-
- ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory,
- valueProviderFactories);
- ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory,
- valueProviderFactories);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
-
- IRTreeInteriorFrame interiorFrame = (IRTreeInteriorFrame) interiorFrameFactory.createFrame();
- IRTreeLeafFrame leafFrame = (IRTreeLeafFrame) leafFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
-
- RTree rtree = new RTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
- rtree.create(fileId);
- rtree.open(fileId);
-
- ByteBuffer hyracksFrame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- DataOutput dos = tb.getDataOutput();
-
- @SuppressWarnings("rawtypes")
- ISerializerDeserializer[] recDescSers = { DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(hyracksFrame);
- FrameTupleReference tuple = new FrameTupleReference();
-
- ITreeIndexAccessor indexAccessor = rtree.createAccessor();
-
- Random rnd = new Random();
- rnd.setSeed(50);
- for (int i = 0; i < 5000; i++) {
-
- double p1x = rnd.nextDouble();
- double p1y = rnd.nextDouble();
- double p2x = rnd.nextDouble();
- double p2y = rnd.nextDouble();
-
- int pk = rnd.nextInt();
-
- tb.reset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(pk, dos);
- tb.addFieldEndOffset();
-
- appender.reset(hyracksFrame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- if (LOGGER.isLoggable(Level.INFO)) {
- if (i % 1000 == 0) {
- LOGGER.info("INSERTING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
- + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
- }
- }
-
- try {
- indexAccessor.insert(tuple);
- } catch (TreeIndexException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- for (int i = 0; i < 50; i++) {
- double p1x = rnd.nextDouble();
- double p1y = rnd.nextDouble();
- double p2x = rnd.nextDouble();
- double p2y = rnd.nextDouble();
-
- int pk = rnd.nextInt();
-
- tb.reset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(pk, dos);
- tb.addFieldEndOffset();
-
- appender.reset(hyracksFrame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(i + " Searching for: " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
- + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
- }
-
- ITreeIndexCursor searchCursor = new RTreeSearchCursor(interiorFrame, leafFrame);
- SearchPredicate searchPredicate = new SearchPredicate(tuple, cmp);
-
- indexAccessor.search(searchCursor, searchPredicate);
-
- ArrayList<Integer> results = new ArrayList<Integer>();
- try {
- while (searchCursor.hasNext()) {
- searchCursor.next();
- ITupleReference frameTuple = searchCursor.getTuple();
- ByteArrayInputStream inStream = new ByteArrayInputStream(frameTuple.getFieldData(4),
- frameTuple.getFieldStart(4), frameTuple.getFieldLength(4));
- DataInput dataIn = new DataInputStream(inStream);
- Integer res = IntegerSerializerDeserializer.INSTANCE.deserialize(dataIn);
- results.add(res);
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- searchCursor.close();
- }
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("There are " + results.size() + " objects that satisfy the query");
- }
- }
-
- rtree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
- }
-}
\ No newline at end of file
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java
new file mode 100644
index 0000000..7520793
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree.multithread;
+
+import java.util.ArrayList;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.ITreeIndexTestWorkerFactory;
+import edu.uci.ics.hyracks.storage.am.common.TestWorkloadConf;
+import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeMultiThreadTest;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
+import edu.uci.ics.hyracks.storage.am.rtree.utils.RTreeTestHarness;
+
+public class RTreeMultiThreadTest extends AbstractRTreeMultiThreadTest {
+
+ private RTreeTestHarness harness = new RTreeTestHarness();
+
+ private RTreeTestWorkerFactory workerFactory = new RTreeTestWorkerFactory();
+
+ @Override
+ protected void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @Override
+ protected void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories)
+ throws TreeIndexException {
+ return RTreeUtils.createRTree(harness.getBufferCache(), typeTraits,
+ valueProviderFactories, rtreeCmpFactories);
+
+ }
+
+ @Override
+ protected ITreeIndexTestWorkerFactory getWorkerFactory() {
+ return workerFactory;
+ }
+
+ @Override
+ protected ArrayList<TestWorkloadConf> getTestWorkloadConf() {
+ ArrayList<TestWorkloadConf> workloadConfs = new ArrayList<TestWorkloadConf>();
+
+ // Insert only workload.
+ TestOperation[] insertOnlyOps = new TestOperation[] { TestOperation.INSERT };
+ workloadConfs.add(new TestWorkloadConf(insertOnlyOps, getUniformOpProbs(insertOnlyOps)));
+
+ // Inserts mixed with scans.
+ TestOperation[] insertSearchOnlyOps = new TestOperation[] { TestOperation.INSERT, TestOperation.SCAN,
+ TestOperation.DISKORDER_SCAN };
+ workloadConfs.add(new TestWorkloadConf(insertSearchOnlyOps, getUniformOpProbs(insertSearchOnlyOps)));
+
+ // Inserts and deletes.
+ TestOperation[] insertDeleteOps = new TestOperation[] { TestOperation.INSERT, TestOperation.DELETE };
+ workloadConfs.add(new TestWorkloadConf(insertDeleteOps, getUniformOpProbs(insertDeleteOps)));
+
+ // All operations mixed.
+ TestOperation[] allOps = new TestOperation[] { TestOperation.INSERT, TestOperation.DELETE, TestOperation.SCAN,
+ TestOperation.DISKORDER_SCAN };
+ workloadConfs.add(new TestWorkloadConf(allOps, getUniformOpProbs(allOps)));
+
+ return workloadConfs;
+ }
+
+ @Override
+ protected int getFileId() {
+ return harness.getTreeFileId();
+ }
+
+ @Override
+ protected String getIndexTypeName() {
+ return "RTree";
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeTestWorker.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeTestWorker.java
new file mode 100644
index 0000000..f5867e6
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeTestWorker.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree.multithread;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.AbstractTreeIndexTestWorker;
+import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector;
+import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector.TestOperation;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.SearchPredicate;
+
+public class RTreeTestWorker extends AbstractTreeIndexTestWorker {
+
+ private final RTree rtree;
+ private final int numFields;
+ private final ArrayTupleReference rearrangedTuple = new ArrayTupleReference();
+ private final ArrayTupleBuilder rearrangedTb;
+
+ public RTreeTestWorker(DataGenThread dataGen, TestOperationSelector opSelector, ITreeIndex index, int numBatches) {
+ super(dataGen, opSelector, index, numBatches);
+ rtree = (RTree) index;
+ numFields = rtree.getFieldCount();
+ rearrangedTb = new ArrayTupleBuilder(numFields);
+ }
+
+ @Override
+ public void performOp(ITupleReference tuple, TestOperation op) throws HyracksDataException, IndexException {
+ RTree.RTreeAccessor accessor = (RTree.RTreeAccessor) indexAccessor;
+ IIndexCursor searchCursor = accessor.createSearchCursor();
+ ITreeIndexCursor diskOrderScanCursor = accessor.createDiskOrderScanCursor();
+ MultiComparator cmp = accessor.getOpContext().cmp;
+ SearchPredicate rangePred = new SearchPredicate(tuple, cmp);
+
+ switch (op) {
+ case INSERT:
+ rearrangeTuple(tuple, cmp);
+ accessor.insert(rearrangedTuple);
+ break;
+
+ case DELETE:
+ rearrangeTuple(tuple, cmp);
+ accessor.delete(rearrangedTuple);
+ break;
+
+ case SCAN:
+ searchCursor.reset();
+ rangePred.setSearchKey(null);
+ accessor.search(searchCursor, rangePred);
+ consumeCursorTuples(searchCursor);
+ break;
+
+ case DISKORDER_SCAN:
+ diskOrderScanCursor.reset();
+ accessor.diskOrderScan(diskOrderScanCursor);
+ consumeCursorTuples(diskOrderScanCursor);
+ break;
+
+ default:
+ throw new HyracksDataException("Op " + op.toString() + " not supported.");
+ }
+ }
+
+ private void rearrangeTuple(ITupleReference tuple, MultiComparator cmp) throws HyracksDataException {
+ // Create a tuple with rearranged key values to make sure lower points
+ // have larger coordinates than high points.
+ rearrangedTb.reset();
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ int c = cmp.getComparators()[i].compare(tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i), tuple.getFieldData(j), tuple.getFieldStart(j), tuple.getFieldLength(j));
+ if (c > 0) {
+ rearrangedTb.addField(tuple.getFieldData(j), tuple.getFieldStart(j), tuple.getFieldLength(j));
+ } else {
+ rearrangedTb.addField(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
+ }
+ }
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ int c = cmp.getComparators()[i].compare(tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i), tuple.getFieldData(j), tuple.getFieldStart(j), tuple.getFieldLength(j));
+ if (c > 0) {
+ rearrangedTb.addField(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
+ } else {
+ rearrangedTb.addField(tuple.getFieldData(j), tuple.getFieldStart(j), tuple.getFieldLength(j));
+ }
+ }
+ for (int i = cmp.getKeyFieldCount(); i < numFields; i++) {
+ rearrangedTb.addField(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
+ }
+ rearrangedTuple.reset(rearrangedTb.getFieldEndOffsets(), rearrangedTb.getByteArray());
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeTestWorkerFactory.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeTestWorkerFactory.java
new file mode 100644
index 0000000..d4f14ca
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeTestWorkerFactory.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree.multithread;
+
+import edu.uci.ics.hyracks.storage.am.common.AbstractTreeIndexTestWorker;
+import edu.uci.ics.hyracks.storage.am.common.ITreeIndexTestWorkerFactory;
+import edu.uci.ics.hyracks.storage.am.common.TestOperationSelector;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.datagen.DataGenThread;
+
+public class RTreeTestWorkerFactory implements ITreeIndexTestWorkerFactory {
+ @Override
+ public AbstractTreeIndexTestWorker create(DataGenThread dataGen, TestOperationSelector opSelector,
+ ITreeIndex index, int numBatches) {
+ return new RTreeTestWorker(dataGen, opSelector, index, numBatches);
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/AbstractRTreeTest.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/AbstractRTreeTest.java
new file mode 100644
index 0000000..fccb29c
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/AbstractRTreeTest.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree.utils;
+
+import java.util.logging.Logger;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public abstract class AbstractRTreeTest {
+ protected final Logger LOGGER = Logger.getLogger(RTreeTestHarness.class.getName());
+ protected final RTreeTestHarness harness;
+
+ public AbstractRTreeTest() {
+ harness = new RTreeTestHarness();
+ }
+
+ public AbstractRTreeTest(int pageSize, int numPages, int maxOpenFiles, int hyracksFrameSize) {
+ harness = new RTreeTestHarness(pageSize, numPages, maxOpenFiles, hyracksFrameSize);
+ }
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/RTreeTestContext.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/RTreeTestContext.java
new file mode 100644
index 0000000..039fb0b
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/RTreeTestContext.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree.utils;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+@SuppressWarnings("rawtypes")
+public class RTreeTestContext extends AbstractRTreeTestContext {
+
+ public RTreeTestContext(ISerializerDeserializer[] fieldSerdes, ITreeIndex treeIndex) {
+ super(fieldSerdes, treeIndex);
+ }
+
+ @Override
+ public int getKeyFieldCount() {
+ RTree rtree = (RTree) treeIndex;
+ return rtree.getComparatorFactories().length;
+ }
+
+ @Override
+ public IBinaryComparatorFactory[] getComparatorFactories() {
+ RTree rtree = (RTree) treeIndex;
+ return rtree.getComparatorFactories();
+ }
+
+ public static RTreeTestContext create(IBufferCache bufferCache, int rtreeFileId,
+ ISerializerDeserializer[] fieldSerdes, IPrimitiveValueProviderFactory[] valueProviderFactories,
+ int numKeyFields) throws Exception {
+ ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
+ IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
+ RTree rtree = RTreeUtils
+ .createRTree(bufferCache, typeTraits, valueProviderFactories, cmpFactories);
+ rtree.create(rtreeFileId);
+ rtree.open(rtreeFileId);
+ RTreeTestContext testCtx = new RTreeTestContext(fieldSerdes, rtree);
+ return testCtx;
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/RTreeTestHarness.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/RTreeTestHarness.java
new file mode 100644
index 0000000..c0cec35
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/utils/RTreeTestHarness.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree.utils;
+
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Random;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
+import edu.uci.ics.hyracks.test.support.TestUtils;
+
+public class RTreeTestHarness {
+
+ private static final long RANDOM_SEED = 50;
+ private static final int DEFAULT_PAGE_SIZE = 256;
+ private static final int DEFAULT_NUM_PAGES = 1000;
+ private static final int DEFAULT_MAX_OPEN_FILES = 10;
+ private static final int DEFAULT_HYRACKS_FRAME_SIZE = 128;
+
+ protected final int pageSize;
+ protected final int numPages;
+ protected final int maxOpenFiles;
+ protected final int hyracksFrameSize;
+
+ protected IHyracksTaskContext ctx;
+ protected IBufferCache bufferCache;
+ protected int treeFileId;
+
+ protected final Random rnd = new Random();
+ protected final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ protected final String tmpDir = System.getProperty("java.io.tmpdir");
+ protected final String sep = System.getProperty("file.separator");
+ protected String fileName;
+
+ public RTreeTestHarness() {
+ this.pageSize = DEFAULT_PAGE_SIZE;
+ this.numPages = DEFAULT_NUM_PAGES;
+ this.maxOpenFiles = DEFAULT_MAX_OPEN_FILES;
+ this.hyracksFrameSize = DEFAULT_HYRACKS_FRAME_SIZE;
+ }
+
+ public RTreeTestHarness(int pageSize, int numPages, int maxOpenFiles, int hyracksFrameSize) {
+ this.pageSize = pageSize;
+ this.numPages = numPages;
+ this.maxOpenFiles = maxOpenFiles;
+ this.hyracksFrameSize = hyracksFrameSize;
+ }
+
+ public void setUp() throws HyracksDataException {
+ fileName = tmpDir + sep + simpleDateFormat.format(new Date());
+ ctx = TestUtils.create(getHyracksFrameSize());
+ TestStorageManagerComponentHolder.init(pageSize, numPages, maxOpenFiles);
+ bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ FileReference file = new FileReference(new File(fileName));
+ bufferCache.createFile(file);
+ treeFileId = fmp.lookupFileId(file);
+ bufferCache.openFile(treeFileId);
+ rnd.setSeed(RANDOM_SEED);
+ }
+
+ public void tearDown() throws HyracksDataException {
+ bufferCache.closeFile(treeFileId);
+ bufferCache.close();
+ File f = new File(fileName);
+ f.deleteOnExit();
+ }
+
+ public IHyracksTaskContext getHyracksTaskContext() {
+ return ctx;
+ }
+
+ public IBufferCache getBufferCache() {
+ return bufferCache;
+ }
+
+ public int getTreeFileId() {
+ return treeFileId;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public Random getRandom() {
+ return rnd;
+ }
+
+ public int getPageSize() {
+ return pageSize;
+ }
+
+ public int getNumPages() {
+ return numPages;
+ }
+
+ public int getHyracksFrameSize() {
+ return hyracksFrameSize;
+ }
+
+ public int getMaxOpenFiles() {
+ return maxOpenFiles;
+ }
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml b/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml
index 6c25b9b..fe652c8 100644
--- a/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml
+++ b/hyracks/hyracks-tests/hyracks-storage-common-test/pom.xml
@@ -33,12 +33,6 @@
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
- <artifactId>hyracks-control-nc</artifactId>
- <version>0.2.1-SNAPSHOT</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-common</artifactId>
<version>0.2.1-SNAPSHOT</version>
<scope>compile</scope>
@@ -48,7 +42,7 @@
<artifactId>hyracks-test-support</artifactId>
<version>0.2.1-SNAPSHOT</version>
<type>jar</type>
- <scope>test</scope>
+ <scope>compile</scope>
</dependency>
</dependencies>
</project>
diff --git a/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/edu/uci/ics/hyracks/storage/common/BufferCacheRegressionTests.java b/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/edu/uci/ics/hyracks/storage/common/BufferCacheRegressionTests.java
index fff9e31..a649aa7 100644
--- a/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/edu/uci/ics/hyracks/storage/common/BufferCacheRegressionTests.java
+++ b/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/edu/uci/ics/hyracks/storage/common/BufferCacheRegressionTests.java
@@ -10,8 +10,8 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileHandle;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IFileHandle;
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.api.io.IIOManager.FileReadWriteMode;
import edu.uci.ics.hyracks.api.io.IIOManager.FileSyncMode;
@@ -23,158 +23,149 @@
import edu.uci.ics.hyracks.test.support.TestUtils;
public class BufferCacheRegressionTests {
- protected static final String tmpDir = System.getProperty("java.io.tmpdir");
- protected static final String sep = System.getProperty("file.separator");
+ protected static final String tmpDir = System.getProperty("java.io.tmpdir");
+ protected static final String sep = System.getProperty("file.separator");
- protected String fileName = tmpDir + sep + "flushTestFile";
+ protected String fileName = tmpDir + sep + "flushTestFile";
- private static final int PAGE_SIZE = 256;
- private static final int HYRACKS_FRAME_SIZE = PAGE_SIZE;
- private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
+ private static final int PAGE_SIZE = 256;
+ private static final int HYRACKS_FRAME_SIZE = PAGE_SIZE;
+ private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
- // We want to test the following behavior when reclaiming a file slot in the
- // buffer cache:
- // 1. If the file being evicted was deleted, then its dirty pages should be
- // invalidated, but most not be flushed.
- // 2. If the file was not deleted, then we must flush its dirty pages.
- @Test
- public void testFlushBehaviorOnFileEviction() throws IOException {
- File f = new File(fileName);
- if (f.exists()) {
- f.delete();
- }
- flushBehaviorTest(true);
- flushBehaviorTest(false);
- }
+ // We want to test the following behavior when reclaiming a file slot in the
+ // buffer cache:
+ // 1. If the file being evicted was deleted, then its dirty pages should be
+ // invalidated, but most not be flushed.
+ // 2. If the file was not deleted, then we must flush its dirty pages.
+ @Test
+ public void testFlushBehaviorOnFileEviction() throws IOException {
+ File f = new File(fileName);
+ if (f.exists()) {
+ f.delete();
+ }
+ flushBehaviorTest(true);
+ flushBehaviorTest(false);
+ }
- private void flushBehaviorTest(boolean deleteFile) throws IOException {
- TestStorageManagerComponentHolder.init(PAGE_SIZE, 10, 1);
+ private void flushBehaviorTest(boolean deleteFile) throws IOException {
+ TestStorageManagerComponentHolder.init(PAGE_SIZE, 10, 1);
- IBufferCache bufferCache = TestStorageManagerComponentHolder
- .getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder
- .getFileMapProvider(ctx);
+ IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference firstFileRef = new FileReference(new File(fileName));
- bufferCache.createFile(firstFileRef);
- int firstFileId = fmp.lookupFileId(firstFileRef);
- bufferCache.openFile(firstFileId);
+ FileReference firstFileRef = new FileReference(new File(fileName));
+ bufferCache.createFile(firstFileRef);
+ int firstFileId = fmp.lookupFileId(firstFileRef);
+ bufferCache.openFile(firstFileId);
- // Fill the first page with known data and make it dirty by write
- // latching it.
- ICachedPage writePage = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(firstFileId, 0), true);
- writePage.acquireWriteLatch();
- try {
- ByteBuffer buf = writePage.getBuffer();
- for (int i = 0; i < buf.capacity(); i++) {
- buf.put(Byte.MAX_VALUE);
- }
- } finally {
- writePage.releaseWriteLatch();
- bufferCache.unpin(writePage);
- }
- bufferCache.closeFile(firstFileId);
- if (deleteFile) {
- bufferCache.deleteFile(firstFileId, false);
- }
+ // Fill the first page with known data and make it dirty by write
+ // latching it.
+ ICachedPage writePage = bufferCache.pin(BufferedFileHandle.getDiskPageId(firstFileId, 0), true);
+ writePage.acquireWriteLatch();
+ try {
+ ByteBuffer buf = writePage.getBuffer();
+ for (int i = 0; i < buf.capacity(); i++) {
+ buf.put(Byte.MAX_VALUE);
+ }
+ } finally {
+ writePage.releaseWriteLatch();
+ bufferCache.unpin(writePage);
+ }
+ bufferCache.closeFile(firstFileId);
+ if (deleteFile) {
+ bufferCache.deleteFile(firstFileId, false);
+ }
- // Create a file with the same name.
- FileReference secondFileRef = new FileReference(new File(fileName));
- bufferCache.createFile(secondFileRef);
- int secondFileId = fmp.lookupFileId(secondFileRef);
+ // Create a file with the same name.
+ FileReference secondFileRef = new FileReference(new File(fileName));
+ bufferCache.createFile(secondFileRef);
+ int secondFileId = fmp.lookupFileId(secondFileRef);
- // This open will replace the firstFileRef's slot in the BufferCache,
- // causing it's pages to be cleaned up. We want to make sure that those
- // dirty pages are not flushed to the disk, because the file was
- // declared as deleted, and
- // somebody might be already using the same filename again (having been
- // assigned a different fileId).
- bufferCache.openFile(secondFileId);
+ // This open will replace the firstFileRef's slot in the BufferCache,
+ // causing it's pages to be cleaned up. We want to make sure that those
+ // dirty pages are not flushed to the disk, because the file was
+ // declared as deleted, and
+ // somebody might be already using the same filename again (having been
+ // assigned a different fileId).
+ bufferCache.openFile(secondFileId);
- // Manually open the file and inspect it's contents. We cannot simply
- // ask the BufferCache to pin the page, because it would return the same
- // physical memory again, and for performance reasons pages are never
- // reset with 0's.
- IIOManager ioManager = ctx.getIOManager();
- FileReference testFileRef = new FileReference(new File(fileName));
- FileHandle testFileHandle = new FileHandle(testFileRef);
- testFileHandle.open(FileReadWriteMode.READ_ONLY,
- FileSyncMode.METADATA_SYNC_DATA_SYNC);
- ByteBuffer testBuffer = ByteBuffer.allocate(PAGE_SIZE);
- ioManager.syncRead(testFileHandle, 0, testBuffer);
- for (int i = 0; i < testBuffer.capacity(); i++) {
- if (deleteFile) {
- // We deleted the file. We expect to see a clean buffer.
- if (testBuffer.get(i) == Byte.MAX_VALUE) {
- fail("Page 0 of deleted file was fazily flushed in openFile(), "
- + "corrupting the data of a newly created file with the same name.");
- }
- } else {
- // We didn't delete the file. We expect to see a buffer full of
- // Byte.MAX_VALUE.
- if (testBuffer.get(i) != Byte.MAX_VALUE) {
- fail("Page 0 of closed file was not flushed when properly, when reclaiming the file slot of fileId 0 in the BufferCache.");
- }
- }
- }
- testFileHandle.close();
- bufferCache.closeFile(secondFileId);
- if (deleteFile) {
- bufferCache.deleteFile(secondFileId, false);
- }
- bufferCache.close();
- }
+ // Manually open the file and inspect it's contents. We cannot simply
+ // ask the BufferCache to pin the page, because it would return the same
+ // physical memory again, and for performance reasons pages are never
+ // reset with 0's.
+ IIOManager ioManager = ctx.getIOManager();
+ FileReference testFileRef = new FileReference(new File(fileName));
+ IFileHandle testFileHandle = ioManager.open(testFileRef, FileReadWriteMode.READ_ONLY,
+ FileSyncMode.METADATA_SYNC_DATA_SYNC);
+ ByteBuffer testBuffer = ByteBuffer.allocate(PAGE_SIZE);
+ ioManager.syncRead(testFileHandle, 0, testBuffer);
+ for (int i = 0; i < testBuffer.capacity(); i++) {
+ if (deleteFile) {
+ // We deleted the file. We expect to see a clean buffer.
+ if (testBuffer.get(i) == Byte.MAX_VALUE) {
+ fail("Page 0 of deleted file was fazily flushed in openFile(), "
+ + "corrupting the data of a newly created file with the same name.");
+ }
+ } else {
+ // We didn't delete the file. We expect to see a buffer full of
+ // Byte.MAX_VALUE.
+ if (testBuffer.get(i) != Byte.MAX_VALUE) {
+ fail("Page 0 of closed file was not flushed when properly, when reclaiming the file slot of fileId 0 in the BufferCache.");
+ }
+ }
+ }
+ ioManager.close(testFileHandle);
+ bufferCache.closeFile(secondFileId);
+ if (deleteFile) {
+ bufferCache.deleteFile(secondFileId, false);
+ }
+ bufferCache.close();
+ }
- // Tests the behavior of the BufferCache when more than all pages are
- // pinned. We expect an exception.
- @Test
- public void testPinningAllPages() throws HyracksDataException {
- int numPages = 10;
- TestStorageManagerComponentHolder.init(PAGE_SIZE, numPages, 1);
+ // Tests the behavior of the BufferCache when more than all pages are
+ // pinned. We expect an exception.
+ @Test
+ public void testPinningAllPages() throws HyracksDataException {
+ int numPages = 10;
+ TestStorageManagerComponentHolder.init(PAGE_SIZE, numPages, 1);
- IBufferCache bufferCache = TestStorageManagerComponentHolder
- .getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder
- .getFileMapProvider(ctx);
+ IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference firstFileRef = new FileReference(new File(fileName));
- bufferCache.createFile(firstFileRef);
- int fileId = fmp.lookupFileId(firstFileRef);
- bufferCache.openFile(fileId);
+ FileReference firstFileRef = new FileReference(new File(fileName));
+ bufferCache.createFile(firstFileRef);
+ int fileId = fmp.lookupFileId(firstFileRef);
+ bufferCache.openFile(fileId);
- // Pin all pages.
- ICachedPage[] pages = new ICachedPage[numPages];
- for (int i = 0; i < numPages; ++i) {
- pages[i] = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, i), true);
- }
+ // Pin all pages.
+ ICachedPage[] pages = new ICachedPage[numPages];
+ for (int i = 0; i < numPages; ++i) {
+ pages[i] = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, i), true);
+ }
- // Try to pin another page. We expect a HyracksDataException.
- ICachedPage errorPage = null;
- try {
- errorPage = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, numPages), true);
- } catch (HyracksDataException e) {
- // This is the expected outcome.
- // The BufferCache should still be able to function properly.
- // Try unpinning all pages.
- for (int i = 0; i < numPages; ++i) {
- bufferCache.unpin(pages[i]);
- }
- // Now try pinning the page that failed above again.
- errorPage = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, numPages), true);
- // Unpin it.
- bufferCache.unpin(errorPage);
- // Cleanup.
- bufferCache.closeFile(fileId);
- bufferCache.close();
- return;
- } catch (Exception e) {
- fail("Expected a HyracksDataException when pinning more pages than available but got another exception: "
- + e.getMessage());
- }
- fail("Expected a HyracksDataException when pinning more pages than available.");
- }
+ // Try to pin another page. We expect a HyracksDataException.
+ ICachedPage errorPage = null;
+ try {
+ errorPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, numPages), true);
+ } catch (HyracksDataException e) {
+ // This is the expected outcome.
+ // The BufferCache should still be able to function properly.
+ // Try unpinning all pages.
+ for (int i = 0; i < numPages; ++i) {
+ bufferCache.unpin(pages[i]);
+ }
+ // Now try pinning the page that failed above again.
+ errorPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, numPages), true);
+ // Unpin it.
+ bufferCache.unpin(errorPage);
+ // Cleanup.
+ bufferCache.closeFile(fileId);
+ bufferCache.close();
+ return;
+ } catch (Exception e) {
+ fail("Expected a HyracksDataException when pinning more pages than available but got another exception: "
+ + e.getMessage());
+ }
+ fail("Expected a HyracksDataException when pinning more pages than available.");
+ }
}
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-am/pom.xml b/hyracks/hyracks-yarn/hyracks-yarn-am/pom.xml
new file mode 100644
index 0000000..9e453a6
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/pom.xml
@@ -0,0 +1,74 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-yarn-am</artifactId>
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-yarn</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>appassembler-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <configuration>
+ <programs>
+ <program>
+ <mainClass>edu.uci.ics.hyracks.yarn.am.HyracksYarnApplicationMaster</mainClass>
+ <name>hyracks-yarn-am</name>
+ </program>
+ </programs>
+ <repositoryLayout>flat</repositoryLayout>
+ <repositoryName>lib</repositoryName>
+ </configuration>
+ <phase>package</phase>
+ <goals>
+ <goal>assemble</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.2-beta-5</version>
+ <executions>
+ <execution>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/binary-assembly.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <phase>package</phase>
+ <goals>
+ <goal>attached</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>args4j</groupId>
+ <artifactId>args4j</artifactId>
+ <version>2.0.16</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-yarn-common</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/assembly/binary-assembly.xml b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/assembly/binary-assembly.xml
new file mode 100644
index 0000000..0500499
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/assembly/binary-assembly.xml
@@ -0,0 +1,19 @@
+<assembly>
+ <id>binary-assembly</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>target/appassembler/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>target/appassembler/lib</directory>
+ <outputDirectory>lib</outputDirectory>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/HyracksYarnApplicationMaster.java b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/HyracksYarnApplicationMaster.java
new file mode 100644
index 0000000..a9c4520
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/HyracksYarnApplicationMaster.java
@@ -0,0 +1,233 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.am;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.util.Records;
+import org.kohsuke.args4j.CmdLineParser;
+
+import edu.uci.ics.hyracks.yarn.am.manifest.AbstractProcess;
+import edu.uci.ics.hyracks.yarn.am.manifest.ContainerSpecification;
+import edu.uci.ics.hyracks.yarn.am.manifest.HyracksCluster;
+import edu.uci.ics.hyracks.yarn.am.manifest.ManifestParser;
+import edu.uci.ics.hyracks.yarn.am.manifest.NodeController;
+import edu.uci.ics.hyracks.yarn.common.protocols.amrm.AMRMConnection;
+
+public class HyracksYarnApplicationMaster {
+ private final Options options;
+
+ private final Timer timer;
+
+ private final List<ResourceRequest> asks;
+
+ private final Map<Resource, Set<AskRecord>> resource2AskMap;
+
+ private final Map<AbstractProcess, AskRecord> proc2AskMap;
+
+ private YarnConfiguration config;
+
+ private AMRMConnection amrmc;
+
+ private RegisterApplicationMasterResponse registration;
+
+ private HyracksCluster hcManifest;
+
+ private HyracksYarnApplicationMaster(Options options) {
+ this.options = options;
+ timer = new Timer(true);
+ asks = new ArrayList<ResourceRequest>();
+ resource2AskMap = new HashMap<Resource, Set<AskRecord>>();
+ proc2AskMap = new HashMap<AbstractProcess, AskRecord>();
+ }
+
+ private void run() throws Exception {
+ Configuration conf = new Configuration();
+ config = new YarnConfiguration(conf);
+ amrmc = new AMRMConnection(config);
+
+ performRegistration();
+ setupHeartbeats();
+ parseManifest();
+ setupAsks();
+ while (true) {
+ Thread.sleep(1000);
+ }
+ }
+
+ private synchronized void setupAsks() {
+ setupAsk(hcManifest.getClusterController());
+ for (NodeController nc : hcManifest.getNodeControllers()) {
+ setupAsk(nc);
+ }
+ }
+
+ private void setupAsk(AbstractProcess proc) {
+ ContainerSpecification cSpec = proc.getContainerSpecification();
+ ResourceRequest rsrcRequest = Records.newRecord(ResourceRequest.class);
+
+ rsrcRequest.setHostName(cSpec.getHostname());
+
+ Priority pri = Records.newRecord(Priority.class);
+ pri.setPriority(100);
+ rsrcRequest.setPriority(pri);
+
+ Resource capability = Records.newRecord(Resource.class);
+ capability.setMemory(cSpec.getMemory());
+ rsrcRequest.setCapability(capability);
+
+ rsrcRequest.setNumContainers(1);
+
+ AskRecord ar = new AskRecord();
+ ar.req = rsrcRequest;
+ ar.proc = proc;
+
+ Set<AskRecord> arSet = resource2AskMap.get(capability);
+ if (arSet == null) {
+ arSet = new HashSet<AskRecord>();
+ resource2AskMap.put(capability, arSet);
+ }
+ arSet.add(ar);
+ proc2AskMap.put(proc, ar);
+
+ System.err.println(proc + " -> " + rsrcRequest);
+
+ asks.add(rsrcRequest);
+ }
+
+ private void parseManifest() throws Exception {
+ String str = FileUtils.readFileToString(new File("manifest.xml"));
+ hcManifest = ManifestParser.parse(str);
+ }
+
+ private void setupHeartbeats() {
+ long heartbeatInterval = config.getLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS);
+ System.err.println("Heartbeat interval: " + heartbeatInterval);
+ heartbeatInterval = Math.min(heartbeatInterval, 1000);
+ System.err.println("Heartbeat interval: " + heartbeatInterval);
+ timer.schedule(new TimerTask() {
+ @Override
+ public void run() {
+ AllocateRequest hb = Records.newRecord(AllocateRequest.class);
+ populateAllocateRequest(hb);
+ hb.setApplicationAttemptId(amrmc.getApplicationAttemptId());
+ hb.setProgress(0);
+ try {
+ AllocateResponse allocateResponse = amrmc.getAMRMProtocol().allocate(hb);
+ List<Container> allocatedContainers = allocateResponse.getAMResponse().getAllocatedContainers();
+ List<ContainerStatus> completedContainers = allocateResponse.getAMResponse()
+ .getCompletedContainersStatuses();
+ processAllocation(allocatedContainers, completedContainers);
+ } catch (YarnRemoteException e) {
+ e.printStackTrace();
+ }
+ }
+ }, 0, heartbeatInterval);
+ }
+
+ private synchronized void populateAllocateRequest(AllocateRequest hb) {
+ hb.addAllAsks(asks);
+ asks.clear();
+ }
+
+ private synchronized void processAllocation(List<Container> allocatedContainers,
+ List<ContainerStatus> completedContainers) {
+ for (Container c : allocatedContainers) {
+ System.err.println("Got container: " + c.getContainerStatus());
+ NodeId nodeId = c.getNodeId();
+ Resource resource = c.getResource();
+
+ Set<AskRecord> arSet = resource2AskMap.get(resource);
+ boolean found = false;
+ if (arSet != null) {
+ AskRecord wildcardMatch = null;
+ AskRecord nameMatch = null;
+ for (AskRecord ar : arSet) {
+ ResourceRequest req = ar.req;
+ if (ar.allocation == null) {
+ if ("*".equals(req.getHostName()) && wildcardMatch == null) {
+ wildcardMatch = ar;
+ }
+ if (req.getHostName().equals(nodeId.getHost()) && nameMatch == null) {
+ nameMatch = ar;
+ break;
+ }
+ }
+ }
+ if (nameMatch != null) {
+ found = true;
+ nameMatch.allocation = c;
+ } else if (wildcardMatch != null) {
+ found = true;
+ wildcardMatch.allocation = c;
+ }
+ }
+ if (!found) {
+ System.err.println("Unknown request satisfied: " + resource);
+ }
+ }
+ }
+
+ private void performRegistration() throws YarnRemoteException {
+ RegisterApplicationMasterRequest appMasterRequest = Records.newRecord(RegisterApplicationMasterRequest.class);
+ appMasterRequest.setApplicationAttemptId(amrmc.getApplicationAttemptId());
+
+ registration = amrmc.getAMRMProtocol().registerApplicationMaster(appMasterRequest);
+ }
+
+ public static void main(String[] args) throws Exception {
+ Options options = new Options();
+ CmdLineParser parser = new CmdLineParser(options);
+ try {
+ parser.parseArgument(args);
+ } catch (Exception e) {
+ parser.printUsage(System.err);
+ return;
+ }
+ new HyracksYarnApplicationMaster(options).run();
+ }
+
+ private static class Options {
+ }
+
+ private static class AskRecord {
+ ResourceRequest req;
+ AbstractProcess proc;
+ Container allocation;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/AbstractProcess.java b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/AbstractProcess.java
new file mode 100644
index 0000000..be6e263
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/AbstractProcess.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.am.manifest;
+
+public abstract class AbstractProcess {
+ protected ContainerSpecification cSpec;
+
+ protected String cmdLineArgs;
+
+ public void setContainerSpecification(ContainerSpecification cSpec) {
+ this.cSpec = cSpec;
+ }
+
+ public ContainerSpecification getContainerSpecification() {
+ return cSpec;
+ }
+
+ public void setCommandLineArguments(String cmdLineArgs) {
+ this.cmdLineArgs = cmdLineArgs;
+ }
+
+ public String getCommandLineArguments() {
+ return cmdLineArgs;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/things/IThingPartition.java b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ClusterController.java
similarity index 84%
rename from hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/things/IThingPartition.java
rename to hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ClusterController.java
index cb79b6f..b230ada 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/things/IThingPartition.java
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ClusterController.java
@@ -12,8 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.api.things;
+package edu.uci.ics.hyracks.yarn.am.manifest;
-public interface IThingPartition {
- public ISliver createSliver();
+public class ClusterController extends AbstractProcess {
}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ContainerSpecification.java b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ContainerSpecification.java
new file mode 100644
index 0000000..07f12c8
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ContainerSpecification.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.am.manifest;
+
+public class ContainerSpecification {
+ private String hostname;
+
+ private int memory;
+
+ public ContainerSpecification() {
+ hostname = "*";
+ }
+
+ public void setHostname(String hostname) {
+ this.hostname = hostname;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public void setMemory(int memory) {
+ this.memory = memory;
+ }
+
+ public int getMemory() {
+ return memory;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/HyracksCluster.java b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/HyracksCluster.java
new file mode 100644
index 0000000..4a66ed6
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/HyracksCluster.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.am.manifest;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class HyracksCluster {
+ private String name;
+
+ private ClusterController cc;
+
+ private List<NodeController> ncs;
+
+ public HyracksCluster() {
+ ncs = new ArrayList<NodeController>();
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setClusterController(ClusterController cc) {
+ this.cc = cc;
+ }
+
+ public ClusterController getClusterController() {
+ return cc;
+ }
+
+ public void addNodeController(NodeController nc) {
+ ncs.add(nc);
+ }
+
+ public List<NodeController> getNodeControllers() {
+ return ncs;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ManifestParser.java b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ManifestParser.java
new file mode 100644
index 0000000..bc5ebae
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/ManifestParser.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.am.manifest;
+
+import java.io.StringReader;
+
+import org.apache.commons.digester.Digester;
+
+public class ManifestParser {
+ public static HyracksCluster parse(String mXML) throws Exception {
+ Digester d = createDigester();
+ return (HyracksCluster) d.parse(new StringReader(mXML));
+ }
+
+ private static Digester createDigester() {
+ Digester d = new Digester();
+ d.setValidating(false);
+
+ d.addObjectCreate("hyracks-cluster", HyracksCluster.class);
+ d.addSetProperties("hyracks-cluster");
+
+ d.addObjectCreate("hyracks-cluster/cluster-controller", ClusterController.class);
+ d.addSetProperties("hyracks-cluster/cluster-controller");
+ d.addSetNext("hyracks-cluster/cluster-controller", "setClusterController");
+
+ d.addObjectCreate("hyracks-cluster/node-controllers/node-controller", NodeController.class);
+ d.addSetProperties("hyracks-cluster/node-controllers/node-controller");
+ d.addSetNext("hyracks-cluster/node-controllers/node-controller", "addNodeController");
+
+ d.addObjectCreate("*/container-specification", ContainerSpecification.class);
+ d.addSetProperties("*/container-specification");
+ d.addSetNext("*/container-specification", "setContainerSpecification");
+ return d;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IReceiverEventListener.java b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/NodeController.java
similarity index 72%
rename from hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IReceiverEventListener.java
rename to hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/NodeController.java
index 98c083e..f46eb7f 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/IReceiverEventListener.java
+++ b/hyracks/hyracks-yarn/hyracks-yarn-am/src/main/java/edu/uci/ics/hyracks/yarn/am/manifest/NodeController.java
@@ -12,10 +12,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hyracks.api.comm;
+package edu.uci.ics.hyracks.yarn.am.manifest;
-public interface IReceiverEventListener {
- public void notifySenderAvailability(int sender);
+public class NodeController extends AbstractProcess {
+ private String id;
- public void notifySenderCount(int senderCount);
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getId() {
+ return id;
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-client/pom.xml b/hyracks/hyracks-yarn/hyracks-yarn-client/pom.xml
new file mode 100644
index 0000000..08935a7
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-client/pom.xml
@@ -0,0 +1,85 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-yarn-client</artifactId>
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-yarn</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>appassembler-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <configuration>
+ <programs>
+ <program>
+ <mainClass>edu.uci.ics.hyracks.yarn.client.LaunchHyracksApplication</mainClass>
+ <name>launch-hyracks-application</name>
+ </program>
+ <program>
+ <mainClass>edu.uci.ics.hyracks.yarn.client.KillHyracksApplication</mainClass>
+ <name>kill-hyracks-application</name>
+ </program>
+ </programs>
+ <repositoryLayout>flat</repositoryLayout>
+ <repositoryName>lib</repositoryName>
+ </configuration>
+ <phase>package</phase>
+ <goals>
+ <goal>assemble</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.2-beta-5</version>
+ <executions>
+ <execution>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/binary-assembly.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <phase>package</phase>
+ <goals>
+ <goal>attached</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>args4j</groupId>
+ <artifactId>args4j</artifactId>
+ <version>2.0.16</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-yarn-common</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-yarn-am</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ <type>zip</type>
+ <classifier>binary-assembly</classifier>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/assembly/binary-assembly.xml b/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/assembly/binary-assembly.xml
new file mode 100644
index 0000000..b4fa339
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/assembly/binary-assembly.xml
@@ -0,0 +1,30 @@
+<assembly>
+ <id>binary-assembly</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>target/appassembler/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>target/appassembler/lib</directory>
+ <outputDirectory>lib</outputDirectory>
+ </fileSet>
+ </fileSets>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>hyracks-yarn-am</outputDirectory>
+ <includes>
+ <include>hyracks-yarn-am*</include>
+ </includes>
+ <unpack>false</unpack>
+ <outputFileNameMapping>${artifact.artifactId}.${artifact.extension}</outputFileNameMapping>
+ <useTransitiveDependencies>false</useTransitiveDependencies>
+ </dependencySet>
+ </dependencySets>
+</assembly>
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/java/edu/uci/ics/hyracks/yarn/client/KillHyracksApplication.java b/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/java/edu/uci/ics/hyracks/yarn/client/KillHyracksApplication.java
new file mode 100644
index 0000000..4789160
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/java/edu/uci/ics/hyracks/yarn/client/KillHyracksApplication.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+
+import edu.uci.ics.hyracks.yarn.common.protocols.clientrm.YarnClientRMConnection;
+
+public class KillHyracksApplication {
+ private final Options options;
+
+ private KillHyracksApplication(Options options) {
+ this.options = options;
+ }
+
+ private void run() throws Exception {
+ Configuration conf = new Configuration();
+ YarnConfiguration yconf = new YarnConfiguration(conf);
+ YarnClientRMConnection crmc = new YarnClientRMConnection(yconf);
+ crmc.killApplication(options.appId);
+ }
+
+ public static void main(String[] args) throws Exception {
+ Options options = new Options();
+ CmdLineParser parser = new CmdLineParser(options);
+ try {
+ parser.parseArgument(args);
+ } catch (Exception e) {
+ parser.printUsage(System.err);
+ return;
+ }
+ new KillHyracksApplication(options).run();
+ }
+
+ private static class Options {
+ @Option(name = "-application-id", required = true, usage = "Application Id")
+ String appId;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/java/edu/uci/ics/hyracks/yarn/client/LaunchHyracksApplication.java b/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/java/edu/uci/ics/hyracks/yarn/client/LaunchHyracksApplication.java
new file mode 100644
index 0000000..e12891e
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-client/src/main/java/edu/uci/ics/hyracks/yarn/client/LaunchHyracksApplication.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.client;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+
+import edu.uci.ics.hyracks.yarn.common.protocols.clientrm.YarnApplication;
+import edu.uci.ics.hyracks.yarn.common.protocols.clientrm.YarnClientRMConnection;
+import edu.uci.ics.hyracks.yarn.common.resources.LocalResourceHelper;
+import edu.uci.ics.hyracks.yarn.common.resources.ResourceHelper;
+
+public class LaunchHyracksApplication {
+ private final Options options;
+
+ private LaunchHyracksApplication(Options options) {
+ this.options = options;
+ }
+
+ private void run() throws Exception {
+ Configuration conf = new Configuration();
+ YarnConfiguration yconf = new YarnConfiguration(conf);
+ YarnClientRMConnection crmc = new YarnClientRMConnection(yconf);
+
+ YarnApplication app = crmc.createApplication(options.appName);
+
+ ContainerLaunchContext clCtx = app.getContainerLaunchContext();
+
+ Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
+
+ File amZipFile = new File(System.getProperty("basedir") + "/hyracks-yarn-am/hyracks-yarn-am.zip");
+ localResources.put("archive", LocalResourceHelper.createArchiveResource(conf, amZipFile));
+ localResources.put("manifest.xml", LocalResourceHelper.createFileResource(conf, options.hcManifest));
+ clCtx.setLocalResources(localResources);
+
+ String command = "./archive/bin/hyracks-yarn-am 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"
+ + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr";
+
+ List<String> commands = new ArrayList<String>();
+ commands.add(command);
+ clCtx.setCommands(commands);
+
+ clCtx.setResource(ResourceHelper.createMemoryCapability(options.amMemory));
+
+ app.submit();
+ }
+
+ public static void main(String[] args) throws Exception {
+ Options options = new Options();
+ CmdLineParser parser = new CmdLineParser(options);
+ try {
+ parser.parseArgument(args);
+ } catch (Exception e) {
+ parser.printUsage(System.err);
+ return;
+ }
+ new LaunchHyracksApplication(options).run();
+ }
+
+ private static class Options {
+ @Option(name = "-application-name", required = true, usage = "Application Name")
+ String appName;
+
+ @Option(name = "-am-memory", required = false, usage = "Application Master memory requirements")
+ int amMemory = 128;
+
+ @Option(name = "-hyracks-cluster-manifest", required = true, usage = "Hyracks Cluster Manifest file")
+ File hcManifest;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/pom.xml b/hyracks/hyracks-yarn/hyracks-yarn-common/pom.xml
new file mode 100644
index 0000000..3aaf4a2
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/pom.xml
@@ -0,0 +1,40 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-yarn-common</artifactId>
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-yarn</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>2.0.0-alpha</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>2.0.0-alpha</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>2.0.0-alpha</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/amrm/AMRMConnection.java b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/amrm/AMRMConnection.java
new file mode 100644
index 0000000..7181bc3
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/amrm/AMRMConnection.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.common.protocols.amrm;
+
+import java.net.InetSocketAddress;
+import java.util.Map;
+
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.AMRMProtocol;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+
+public class AMRMConnection {
+ private final YarnConfiguration config;
+
+ private final ApplicationAttemptId appAttemptId;
+
+ private final AMRMProtocol amrmp;
+
+ public AMRMConnection(YarnConfiguration config) {
+ this.config = config;
+ Map<String, String> envs = System.getenv();
+ String containerIdString = envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV);
+ if (containerIdString == null) {
+ throw new IllegalArgumentException("ContainerId not set in the environment");
+ }
+ ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
+ appAttemptId = containerId.getApplicationAttemptId();
+ InetSocketAddress rmAddress = NetUtils.createSocketAddr(config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS));
+ YarnRPC rpc = YarnRPC.create(config);
+
+ amrmp = (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, config);
+ }
+
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return appAttemptId;
+ }
+
+ public AMRMProtocol getAMRMProtocol() {
+ return amrmp;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnApplication.java b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnApplication.java
new file mode 100644
index 0000000..ca099aa
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnApplication.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.common.protocols.clientrm;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.util.Records;
+
+public class YarnApplication {
+ private final YarnClientRMConnection crmc;
+
+ private ApplicationSubmissionContext appCtx;
+
+ private ContainerLaunchContext clCtx;
+
+ YarnApplication(YarnClientRMConnection crmc, String appName) throws YarnRemoteException {
+ this.crmc = crmc;
+ appCtx = Records.newRecord(ApplicationSubmissionContext.class);
+ appCtx.setApplicationId(getNewApplicationId(crmc));
+ appCtx.setApplicationName(appName);
+ clCtx = Records.newRecord(ContainerLaunchContext.class);
+ }
+
+ public ContainerLaunchContext getContainerLaunchContext() {
+ return clCtx;
+ }
+
+ public void submit() throws YarnRemoteException {
+ appCtx.setAMContainerSpec(clCtx);
+ SubmitApplicationRequest appRequest = Records.newRecord(SubmitApplicationRequest.class);
+ appRequest.setApplicationSubmissionContext(appCtx);
+ crmc.getClientRMProtocol().submitApplication(appRequest);
+ }
+
+ private static ApplicationId getNewApplicationId(YarnClientRMConnection crmc) throws YarnRemoteException {
+ GetNewApplicationRequest request = Records.newRecord(GetNewApplicationRequest.class);
+ GetNewApplicationResponse response = crmc.getClientRMProtocol().getNewApplication(request);
+
+ return response.getApplicationId();
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnClientRMConnection.java b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnClientRMConnection.java
new file mode 100644
index 0000000..bb42fab
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/protocols/clientrm/YarnClientRMConnection.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.common.protocols.clientrm;
+
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityInfo;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo;
+import org.apache.hadoop.yarn.util.Records;
+
+public class YarnClientRMConnection {
+ private final YarnConfiguration config;
+
+ private final ClientRMProtocol crmp;
+
+ public YarnClientRMConnection(YarnConfiguration config) {
+ this.config = config;
+ InetSocketAddress remoteAddress = NetUtils.createSocketAddr(config.get(YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS));
+ Configuration appsManagerServerConf = new Configuration(config);
+ appsManagerServerConf.setClass(YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_CLIENT_RESOURCEMANAGER,
+ ClientRMSecurityInfo.class, SecurityInfo.class);
+ YarnRPC rpc = YarnRPC.create(appsManagerServerConf);
+ crmp = ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, remoteAddress, appsManagerServerConf));
+ }
+
+ public YarnApplication createApplication(String appName) throws YarnRemoteException {
+ return new YarnApplication(this, appName);
+ }
+
+ public ClientRMProtocol getClientRMProtocol() {
+ return crmp;
+ }
+
+ public void killApplication(String appId) throws Exception {
+ KillApplicationRequest killRequest = Records.newRecord(KillApplicationRequest.class);
+ ApplicationId aid = Records.newRecord(ApplicationId.class);
+ long ts = Long.parseLong(appId.substring(appId.indexOf('_') + 1, appId.lastIndexOf('_')));
+ aid.setClusterTimestamp(ts);
+ int id = Integer.parseInt(appId.substring(appId.lastIndexOf('_') + 1));
+ aid.setId(id);
+ killRequest.setApplicationId(aid);
+ crmp.forceKillApplication(killRequest);
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/LocalResourceHelper.java b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/LocalResourceHelper.java
new file mode 100644
index 0000000..b30c4a1
--- /dev/null
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/LocalResourceHelper.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.yarn.common.resources;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
+
+public class LocalResourceHelper {
+ private static LocalResource createLocalResourceFromPath(Configuration config, File path) throws IOException {
+ LocalResource lr = Records.newRecord(LocalResource.class);
+ URL url = ConverterUtils.getYarnUrlFromPath(FileContext.getFileContext().makeQualified(new Path(path.toURI())));
+ lr.setResource(url);
+ lr.setVisibility(LocalResourceVisibility.APPLICATION);
+ lr.setTimestamp(path.lastModified());
+ lr.setSize(path.length());
+ return lr;
+ }
+
+ public static LocalResource createFileResource(Configuration config, File path) throws IOException {
+ LocalResource lr = createLocalResourceFromPath(config, path);
+ lr.setType(LocalResourceType.FILE);
+ return lr;
+ }
+
+ public static LocalResource createArchiveResource(Configuration config, File path) throws IOException {
+ LocalResource lr = createLocalResourceFromPath(config, path);
+ lr.setType(LocalResourceType.ARCHIVE);
+ return lr;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/PageAllocationException.java b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/ResourceHelper.java
similarity index 63%
copy from hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/PageAllocationException.java
copy to hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/ResourceHelper.java
index e6eec66..20267b7 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/PageAllocationException.java
+++ b/hyracks/hyracks-yarn/hyracks-yarn-common/src/main/java/edu/uci/ics/hyracks/yarn/common/resources/ResourceHelper.java
@@ -12,18 +12,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package edu.uci.ics.hyracks.yarn.common.resources;
-package edu.uci.ics.hyracks.storage.am.common.api;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Records;
-public class PageAllocationException extends Exception {
-
- private static final long serialVersionUID = 1L;
-
- public PageAllocationException(Throwable cause) {
- super(cause);
+public class ResourceHelper {
+ public static Resource createMemoryCapability(int memory) {
+ Resource capability = Records.newRecord(Resource.class);
+ capability.setMemory(memory);
+ return capability;
}
-
- public PageAllocationException(String message) {
- super(message);
- }
-}
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-yarn/pom.xml b/hyracks/hyracks-yarn/pom.xml
new file mode 100644
index 0000000..af4b0b1
--- /dev/null
+++ b/hyracks/hyracks-yarn/pom.xml
@@ -0,0 +1,17 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-yarn</artifactId>
+ <packaging>pom</packaging>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </parent>
+
+ <modules>
+ <module>hyracks-yarn-common</module>
+ <module>hyracks-yarn-client</module>
+ <module>hyracks-yarn-am</module>
+ </modules>
+</project>
diff --git a/hyracks/pom.xml b/hyracks/pom.xml
index 9d5c851..77c32c4 100644
--- a/hyracks/pom.xml
+++ b/hyracks/pom.xml
@@ -1,3 +1,4 @@
+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
@@ -5,6 +6,25 @@
<version>0.2.1-SNAPSHOT</version>
<packaging>pom</packaging>
+ <properties>
+ <jvm.extraargs></jvm.extraargs>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>macosx</id>
+ <activation>
+ <os>
+ <name>mac os x</name>
+ </os>
+ <jdk>1.7</jdk>
+ </activation>
+ <properties>
+ <jvm.extraargs>-Djava.nio.channels.spi.SelectorProvider=sun.nio.ch.KQueueSelectorProvider</jvm.extraargs>
+ </properties>
+ </profile>
+ </profiles>
+
<build>
<plugins>
<plugin>
@@ -25,7 +45,7 @@
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkMode>pertest</forkMode>
- <argLine>-enableassertions -Djava.util.logging.config.file=${user.home}/logging.properties -Xdebug -Xrunjdwp:transport=dt_socket,server=y,address=8000,suspend=n</argLine>
+ <argLine>-enableassertions -Djava.util.logging.config.file=${user.home}/logging.properties -Xdebug -Xrunjdwp:transport=dt_socket,server=y,address=8000,suspend=n ${jvm.extraargs}</argLine>
</configuration>
</plugin>
</plugins>
@@ -100,5 +120,7 @@
<module>hyracks-documentation</module>
<module>hyracks-hadoop-compat</module>
<module>hyracks-algebricks</module>
+ <!--module>hyracks-yarn</module-->
+ <module>hyracks-maven-plugins</module>
</modules>
</project>