Merge branch 'master' of https://code.google.com/p/hyracks into ceej/tiered-tests
diff --git a/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java b/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
index 39587b9..26dc343 100644
--- a/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
+++ b/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.PlanCompiler;
@@ -47,9 +48,10 @@
IMergeAggregationExpressionFactory mergeAggregationExpressionFactory,
IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer,
PhysicalOptimizationConfig physicalOptimizationConfig) {
+ LogicalOperatorPrettyPrintVisitor prettyPrintVisitor = new LogicalOperatorPrettyPrintVisitor();
return new AlgebricksOptimizationContext(varCounter, expressionEvalSizeComputer,
mergeAggregationExpressionFactory, expressionTypeComputer, nullableTypeComputer,
- physicalOptimizationConfig);
+ physicalOptimizationConfig, prettyPrintVisitor);
}
}
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IOptimizationContext.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
index 8537702..9b9273a 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
@@ -22,6 +22,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableEvalSizeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
@@ -84,4 +85,6 @@
public abstract void computeAndSetTypeEnvironmentForOperator(ILogicalOperator op) throws AlgebricksException;
public abstract void updatePrimaryKeys(Map<LogicalVariable, LogicalVariable> mappedVars);
+
+ public abstract LogicalOperatorPrettyPrintVisitor getPrettyPrintVisitor();
}
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalExpressionPrettyPrintVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalExpressionPrettyPrintVisitor.java
new file mode 100644
index 0000000..5f77352
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalExpressionPrettyPrintVisitor.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2014 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+
+
+public class LogicalExpressionPrettyPrintVisitor implements ILogicalExpressionVisitor<String, Integer> {
+
+ @Override
+ public String visitConstantExpression(ConstantExpression expr, Integer indent)
+ throws AlgebricksException {
+ return expr.toString();
+ }
+
+ @Override
+ public String visitVariableReferenceExpression(
+ VariableReferenceExpression expr, Integer indent)
+ throws AlgebricksException {
+ return expr.toString();
+ }
+
+ @Override
+ public String visitAggregateFunctionCallExpression(
+ AggregateFunctionCallExpression expr, Integer indent)
+ throws AlgebricksException {
+ return expr.toString();
+ }
+
+ @Override
+ public String visitScalarFunctionCallExpression(
+ ScalarFunctionCallExpression expr, Integer indent)
+ throws AlgebricksException {
+ return expr.toString();
+ }
+
+ @Override
+ public String visitStatefulFunctionCallExpression(
+ StatefulFunctionCallExpression expr, Integer indent)
+ throws AlgebricksException {
+ return expr.toString();
+ }
+
+ @Override
+ public String visitUnnestingFunctionCallExpression(
+ UnnestingFunctionCallExpression expr, Integer indent)
+ throws AlgebricksException {
+ return expr.toString();
+ }
+}
+
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
index 49ec269..5a2619d 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
@@ -55,26 +55,34 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
public class LogicalOperatorPrettyPrintVisitor implements ILogicalOperatorVisitor<String, Integer> {
+ ILogicalExpressionVisitor<String, Integer> exprVisitor;
+
public LogicalOperatorPrettyPrintVisitor() {
+ exprVisitor = new LogicalExpressionPrettyPrintVisitor();
+ }
+
+ public LogicalOperatorPrettyPrintVisitor(ILogicalExpressionVisitor<String, Integer> exprVisitor) {
+ this.exprVisitor = exprVisitor;
}
@Override
- public String visitAggregateOperator(AggregateOperator op, Integer indent) {
+ public String visitAggregateOperator(AggregateOperator op, Integer indent) throws AlgebricksException {
StringBuilder buffer = new StringBuilder();
addIndent(buffer, indent).append("aggregate ").append(op.getVariables()).append(" <- ");
- pprintExprList(op.getExpressions(), buffer);
+ pprintExprList(op.getExpressions(), buffer, indent);
return buffer.toString();
}
@Override
- public String visitRunningAggregateOperator(RunningAggregateOperator op, Integer indent) {
+ public String visitRunningAggregateOperator(RunningAggregateOperator op, Integer indent) throws AlgebricksException {
StringBuilder buffer = new StringBuilder();
addIndent(buffer, indent).append("running-aggregate ").append(op.getVariables()).append(" <- ");
- pprintExprList(op.getExpressions(), buffer);
+ pprintExprList(op.getExpressions(), buffer, indent);
return buffer.toString();
}
@@ -88,32 +96,37 @@
@Override
public String visitGroupByOperator(GroupByOperator op, Integer indent) throws AlgebricksException {
StringBuilder buffer = new StringBuilder();
- addIndent(buffer, indent).append("group by (").append(op.gByListToString()).append(") decor (")
- .append(op.decorListToString()).append(") {");
+ addIndent(buffer, indent).append("group by (");
+ pprintVeList(buffer, op.getGroupByList(), indent);
+ buffer.append(") decor (");
+ pprintVeList(buffer, op.getDecorList(), indent);
+ buffer.append(") {");
printNestedPlans(op, indent, buffer);
return buffer.toString();
}
@Override
- public String visitDistinctOperator(DistinctOperator op, Integer indent) {
+ public String visitDistinctOperator(DistinctOperator op, Integer indent) throws AlgebricksException {
StringBuilder buffer = new StringBuilder();
addIndent(buffer, indent).append("distinct " + "(");
- pprintExprList(op.getExpressions(), buffer);
+ pprintExprList(op.getExpressions(), buffer, indent);
buffer.append(")");
return buffer.toString();
}
@Override
- public String visitInnerJoinOperator(InnerJoinOperator op, Integer indent) {
+ public String visitInnerJoinOperator(InnerJoinOperator op, Integer indent) throws AlgebricksException {
StringBuilder buffer = new StringBuilder();
- addIndent(buffer, indent).append("join (").append(op.getCondition().getValue()).append(")");
+ addIndent(buffer, indent).append("join (").append(op.getCondition().getValue().accept(exprVisitor, indent))
+ .append(")");
return buffer.toString();
}
@Override
- public String visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Integer indent) {
+ public String visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Integer indent) throws AlgebricksException {
StringBuilder buffer = new StringBuilder();
- addIndent(buffer, indent).append("left outer join (").append(op.getCondition().getValue()).append(")");
+ addIndent(buffer, indent).append("left outer join (")
+ .append(op.getCondition().getValue().accept(exprVisitor, indent)).append(")");
return buffer.toString();
}
@@ -125,7 +138,7 @@
}
@Override
- public String visitOrderOperator(OrderOperator op, Integer indent) {
+ public String visitOrderOperator(OrderOperator op, Integer indent) throws AlgebricksException {
StringBuilder buffer = new StringBuilder();
addIndent(buffer, indent).append("order ");
for (Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>> p : op.getOrderExpressions()) {
@@ -143,45 +156,49 @@
fst = p.first.getExpressionRef().toString();
}
}
- buffer.append("(" + fst + ", " + p.second.getValue() + ") ");
+ buffer.append("(" + fst + ", " + p.second.getValue().accept(exprVisitor, indent) + ") ");
}
return buffer.toString();
}
@Override
- public String visitAssignOperator(AssignOperator op, Integer indent) {
+ public String visitAssignOperator(AssignOperator op, Integer indent) throws AlgebricksException {
StringBuilder buffer = new StringBuilder();
addIndent(buffer, indent).append("assign ").append(op.getVariables()).append(" <- ");
- pprintExprList(op.getExpressions(), buffer);
+ pprintExprList(op.getExpressions(), buffer, indent);
return buffer.toString();
}
@Override
- public String visitWriteOperator(WriteOperator op, Integer indent) {
+ public String visitWriteOperator(WriteOperator op, Integer indent) throws AlgebricksException {
StringBuilder buffer = new StringBuilder();
- addIndent(buffer, indent).append("write ").append(op.getExpressions());
+ addIndent(buffer, indent).append("write ");
+ pprintExprList(op.getExpressions(), buffer, indent);
return buffer.toString();
}
@Override
- public String visitDistributeResultOperator(DistributeResultOperator op, Integer indent) {
+ public String visitDistributeResultOperator(DistributeResultOperator op, Integer indent) throws AlgebricksException {
StringBuilder buffer = new StringBuilder();
- addIndent(buffer, indent).append("distribute result ").append(op.getExpressions());
+ addIndent(buffer, indent).append("distribute result ");
+ pprintExprList(op.getExpressions(), buffer, indent);
return buffer.toString();
}
@Override
- public String visitWriteResultOperator(WriteResultOperator op, Integer indent) {
+ public String visitWriteResultOperator(WriteResultOperator op, Integer indent) throws AlgebricksException {
StringBuilder buffer = new StringBuilder();
addIndent(buffer, indent).append("load ").append(op.getDataSource()).append(" from ")
- .append(op.getPayloadExpression()).append(" partitioned by ").append(op.getKeyExpressions().toString());
+ .append(op.getPayloadExpression().getValue().accept(exprVisitor, indent)).append(" partitioned by ");
+ pprintExprList(op.getKeyExpressions(), buffer, indent);
return buffer.toString();
}
@Override
- public String visitSelectOperator(SelectOperator op, Integer indent) {
+ public String visitSelectOperator(SelectOperator op, Integer indent) throws AlgebricksException {
StringBuilder buffer = new StringBuilder();
- addIndent(buffer, indent).append("select " + "(" + op.getCondition().getValue() + ")");
+ addIndent(buffer, indent).append("select (").append(op.getCondition().getValue().accept(exprVisitor, indent))
+ .append(")");
return buffer.toString();
}
@@ -193,9 +210,12 @@
}
@Override
- public String visitPartitioningSplitOperator(PartitioningSplitOperator op, Integer indent) {
+ public String visitPartitioningSplitOperator(PartitioningSplitOperator op, Integer indent)
+ throws AlgebricksException {
StringBuilder buffer = new StringBuilder();
- addIndent(buffer, indent).append("partitioning-split (" + op.getExpressions() + ")");
+ addIndent(buffer, indent).append("partitioning-split (");
+ pprintExprList(op.getExpressions(), buffer, indent);
+ buffer.append(")");
return buffer.toString();
}
@@ -218,20 +238,22 @@
}
@Override
- public String visitUnnestOperator(UnnestOperator op, Integer indent) {
+ public String visitUnnestOperator(UnnestOperator op, Integer indent) throws AlgebricksException {
StringBuilder buffer = new StringBuilder();
addIndent(buffer, indent).append("unnest " + op.getVariable());
if (op.getPositionalVariable() != null) {
buffer.append(" at " + op.getPositionalVariable());
}
- buffer.append(" <- " + op.getExpressionRef().getValue());
+ buffer.append(" <- " + op.getExpressionRef().getValue().accept(exprVisitor, indent));
return buffer.toString();
}
@Override
- public String visitUnnestMapOperator(UnnestMapOperator op, Integer indent) {
+ public String visitUnnestMapOperator(UnnestMapOperator op, Integer indent) throws AlgebricksException {
StringBuilder buffer = new StringBuilder();
- addIndent(buffer, indent).append("unnest-map " + op.getVariables() + " <- " + op.getExpressionRef().getValue());
+ addIndent(buffer, indent).append(
+ "unnest-map " + op.getVariables() + " <- "
+ + op.getExpressionRef().getValue().accept(exprVisitor, indent));
return buffer.toString();
}
@@ -244,12 +266,12 @@
}
@Override
- public String visitLimitOperator(LimitOperator op, Integer indent) {
+ public String visitLimitOperator(LimitOperator op, Integer indent) throws AlgebricksException {
StringBuilder buffer = new StringBuilder();
- addIndent(buffer, indent).append("limit " + op.getMaxObjects().getValue());
+ addIndent(buffer, indent).append("limit " + op.getMaxObjects().getValue().accept(exprVisitor, indent));
ILogicalExpression offset = op.getOffset().getValue();
if (offset != null) {
- buffer.append(", " + offset);
+ buffer.append(", " + offset.accept(exprVisitor, indent));
}
return buffer.toString();
}
@@ -261,6 +283,57 @@
return buffer.toString();
}
+ @Override
+ public String visitScriptOperator(ScriptOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append(
+ "script (in: " + op.getInputVariables() + ") (out: " + op.getOutputVariables() + ")");
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitReplicateOperator(ReplicateOperator op, Integer indent) throws AlgebricksException {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("replicate ");
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitInsertDeleteOperator(InsertDeleteOperator op, Integer indent) throws AlgebricksException {
+ StringBuilder buffer = new StringBuilder();
+ String header = op.getOperation() == Kind.INSERT ? "insert into " : "delete from ";
+ addIndent(buffer, indent).append(header).append(op.getDataSource()).append(" from ")
+ .append(op.getPayloadExpression().getValue().accept(exprVisitor, indent)).append(" partitioned by ");
+ pprintExprList(op.getPrimaryKeyExpressions(), buffer, indent);
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Integer indent)
+ throws AlgebricksException {
+ StringBuilder buffer = new StringBuilder();
+ String header = op.getOperation() == Kind.INSERT ? "insert into " : "delete from ";
+ addIndent(buffer, indent).append(header).append(op.getDataSourceIndex()).append(" from ");
+ pprintExprList(op.getSecondaryKeyExpressions(), buffer, indent);
+ buffer.append(" ");
+ pprintExprList(op.getPrimaryKeyExpressions(), buffer, indent);
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitSinkOperator(SinkOperator op, Integer indent) throws AlgebricksException {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("sink");
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitExtensionOperator(ExtensionOperator op, Integer indent) throws AlgebricksException {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append(op.toString());
+ return buffer.toString();
+ }
+
protected static final StringBuilder addIndent(StringBuilder buffer, int level) {
for (int i = 0; i < level; ++i) {
buffer.append(' ');
@@ -268,7 +341,7 @@
return buffer;
}
- private void printNestedPlans(AbstractOperatorWithNestedPlans op, Integer indent, StringBuilder buffer)
+ protected void printNestedPlans(AbstractOperatorWithNestedPlans op, Integer indent, StringBuilder buffer)
throws AlgebricksException {
boolean first = true;
if (op.getNestedPlans().isEmpty()) {
@@ -289,15 +362,8 @@
}
}
- @Override
- public String visitScriptOperator(ScriptOperator op, Integer indent) {
- StringBuilder buffer = new StringBuilder();
- addIndent(buffer, indent).append(
- "script (in: " + op.getInputVariables() + ") (out: " + op.getOutputVariables() + ")");
- return buffer.toString();
- }
-
- private void pprintExprList(List<Mutable<ILogicalExpression>> expressions, StringBuilder buffer) {
+ protected void pprintExprList(List<Mutable<ILogicalExpression>> expressions, StringBuilder buffer, Integer indent)
+ throws AlgebricksException {
buffer.append("[");
boolean first = true;
for (Mutable<ILogicalExpression> exprRef : expressions) {
@@ -306,51 +372,28 @@
} else {
buffer.append(", ");
}
- buffer.append(exprRef.getValue());
+ buffer.append(exprRef.getValue().accept(exprVisitor, indent));
}
buffer.append("]");
}
- @Override
- public String visitReplicateOperator(ReplicateOperator op, Integer indent) throws AlgebricksException {
- StringBuilder buffer = new StringBuilder();
- addIndent(buffer, indent).append("replicate ");
- return buffer.toString();
- }
-
- @Override
- public String visitInsertDeleteOperator(InsertDeleteOperator op, Integer indent) throws AlgebricksException {
- StringBuilder buffer = new StringBuilder();
- String header = op.getOperation() == Kind.INSERT ? "insert into " : "delete from ";
- addIndent(buffer, indent).append(header).append(op.getDataSource()).append(" from ")
- .append(op.getPayloadExpression()).append(" partitioned by ")
- .append(op.getPrimaryKeyExpressions().toString());
- return buffer.toString();
- }
-
- @Override
- public String visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Integer indent)
- throws AlgebricksException {
- StringBuilder buffer = new StringBuilder();
- String header = op.getOperation() == Kind.INSERT ? "insert into " : "delete from ";
- addIndent(buffer, indent).append(header).append(op.getDataSourceIndex()).append(" from ")
- .append(op.getSecondaryKeyExpressions().toString()).append(" ")
- .append(op.getPrimaryKeyExpressions().toString());
- return buffer.toString();
- }
-
- @Override
- public String visitSinkOperator(SinkOperator op, Integer indent) throws AlgebricksException {
- StringBuilder buffer = new StringBuilder();
- addIndent(buffer, indent).append("sink");
- return buffer.toString();
- }
-
- @Override
- public String visitExtensionOperator(ExtensionOperator op, Integer indent) throws AlgebricksException {
- StringBuilder buffer = new StringBuilder();
- addIndent(buffer, indent).append(op.toString());
- return buffer.toString();
+ protected void pprintVeList(StringBuilder sb, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> vePairList,
+ Integer indent) throws AlgebricksException {
+ sb.append("[");
+ boolean fst = true;
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : vePairList) {
+ if (fst) {
+ fst = false;
+ } else {
+ sb.append("; ");
+ }
+ if (ve.first != null) {
+ sb.append(ve.first + " := " + ve.second);
+ } else {
+ sb.append(ve.second.getValue().accept(exprVisitor, indent));
+ }
+ }
+ sb.append("]");
}
}
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java
index 77d8d40..361dad3 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java
@@ -32,6 +32,7 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableEvalSizeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
@@ -72,17 +73,27 @@
protected final Map<ILogicalOperator, ILogicalPropertiesVector> logicalProps = new HashMap<ILogicalOperator, ILogicalPropertiesVector>();
private final IExpressionTypeComputer expressionTypeComputer;
private final INullableTypeComputer nullableTypeComputer;
+ private final LogicalOperatorPrettyPrintVisitor prettyPrintVisitor;
public AlgebricksOptimizationContext(int varCounter, IExpressionEvalSizeComputer expressionEvalSizeComputer,
IMergeAggregationExpressionFactory mergeAggregationExpressionFactory,
IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer,
PhysicalOptimizationConfig physicalOptimizationConfig) {
+ this(varCounter, expressionEvalSizeComputer, mergeAggregationExpressionFactory, expressionTypeComputer,
+ nullableTypeComputer, physicalOptimizationConfig, new LogicalOperatorPrettyPrintVisitor());
+ }
+
+ public AlgebricksOptimizationContext(int varCounter, IExpressionEvalSizeComputer expressionEvalSizeComputer,
+ IMergeAggregationExpressionFactory mergeAggregationExpressionFactory,
+ IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer,
+ PhysicalOptimizationConfig physicalOptimizationConfig, LogicalOperatorPrettyPrintVisitor prettyPrintVisitor) {
this.varCounter = varCounter;
this.expressionEvalSizeComputer = expressionEvalSizeComputer;
this.mergeAggregationExpressionFactory = mergeAggregationExpressionFactory;
this.expressionTypeComputer = expressionTypeComputer;
this.nullableTypeComputer = nullableTypeComputer;
this.physicalOptimizationConfig = physicalOptimizationConfig;
+ this.prettyPrintVisitor = prettyPrintVisitor;
}
public int getVarCounter() {
@@ -280,4 +291,9 @@
me.setValue(new FunctionalDependency(hd, tl));
}
}
+
+ @Override
+ public LogicalOperatorPrettyPrintVisitor getPrettyPrintVisitor() {
+ return prettyPrintVisitor;
+ }
}
\ No newline at end of file
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
index f684c8a..9035ca5 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
@@ -54,7 +54,6 @@
private List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> logicalRewrites;
private List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> physicalRewrites;
private ILogicalPlan plan;
- private LogicalOperatorPrettyPrintVisitor ppvisitor = new LogicalOperatorPrettyPrintVisitor();
public HeuristicOptimizer(ILogicalPlan plan,
List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> logicalRewrites,
@@ -75,13 +74,13 @@
}
StringBuilder sb = new StringBuilder();
- PlanPrettyPrinter.printPlan(plan, sb, ppvisitor, 0);
+ PlanPrettyPrinter.printPlan(plan, sb, context.getPrettyPrintVisitor(), 0);
AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Logical Plan:\n" + sb.toString());
runOptimizationSets(plan, logicalRewrites);
computeSchemaBottomUpForPlan(plan);
runPhysicalOptimizations(plan, physicalRewrites);
StringBuilder sb2 = new StringBuilder();
- PlanPrettyPrinter.printPlan(plan, sb2, ppvisitor, 0);
+ PlanPrettyPrinter.printPlan(plan, sb2, context.getPrettyPrintVisitor(), 0);
AlgebricksConfig.ALGEBRICKS_LOGGER.info("Optimized Plan:\n" + sb2.toString());
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/PrefixMergePolicy.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/PrefixMergePolicy.java
index fe04db6..2e2f084 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/PrefixMergePolicy.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/PrefixMergePolicy.java
@@ -16,6 +16,7 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -31,7 +32,7 @@
public class PrefixMergePolicy implements ILSMMergePolicy {
private long maxMergableComponentSize;
- private int maxTolernaceComponentCount;
+ private int maxToleranceComponentCount;
@Override
public void diskComponentAdded(final ILSMIndex index, boolean fullMergeIsRequested) throws HyracksDataException,
@@ -40,7 +41,10 @@
// all such components for which the sum of their sizes exceeds MaxMrgCompSz. Schedule a merge of those components into a new component.
// 2. If a merge from 1 doesn't happen, see if the set of candidate components for merging exceeds MaxTolCompCnt. If so, schedule
// a merge all of the current candidates into a new single component.
- List<ILSMComponent> immutableComponents = index.getImmutableComponents();
+ List<ILSMComponent> immutableComponents = new ArrayList<ILSMComponent>(index.getImmutableComponents());
+ // Reverse the components order so that we look at components from oldest to newest.
+ Collections.reverse(immutableComponents);
+
for (ILSMComponent c : immutableComponents) {
if (c.getState() != ComponentState.READABLE_UNWRITABLE) {
return;
@@ -65,14 +69,16 @@
totalSize += componentSize;
boolean isLastComponent = i + 1 == immutableComponents.size() ? true : false;
if (totalSize > maxMergableComponentSize
- || (isLastComponent && i - startIndex >= maxTolernaceComponentCount)) {
- List<ILSMComponent> mergableCopments = new ArrayList<ILSMComponent>();
+ || (isLastComponent && i - startIndex >= maxToleranceComponentCount)) {
+ List<ILSMComponent> mergableComponents = new ArrayList<ILSMComponent>();
for (int j = startIndex + 1; j <= i; j++) {
- mergableCopments.add(immutableComponents.get(j));
+ mergableComponents.add(immutableComponents.get(j));
}
+ // Reverse the components order back to its original order
+ Collections.reverse(mergableComponents);
ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
- accessor.scheduleMerge(index.getIOOperationCallback(), mergableCopments);
+ accessor.scheduleMerge(index.getIOOperationCallback(), mergableComponents);
break;
}
}
@@ -81,6 +87,6 @@
@Override
public void configure(Map<String, String> properties) {
maxMergableComponentSize = Long.parseLong(properties.get("max-mergable-component-size"));
- maxTolernaceComponentCount = Integer.parseInt(properties.get("max-tolernace-component-count"));
+ maxToleranceComponentCount = Integer.parseInt(properties.get("max-tolerance-component-count"));
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/PrefixMergePolicyFactory.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/PrefixMergePolicyFactory.java
index 981ec6c..8ffe1b0 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/PrefixMergePolicyFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/PrefixMergePolicyFactory.java
@@ -28,7 +28,7 @@
private static final long serialVersionUID = 1L;
private static final String[] SET_VALUES = new String[] { "max-mergable-component-size",
- "max-tolernace-component-count" };
+ "max-tolerance-component-count" };
private static final Set<String> PROPERTIES_NAMES = new HashSet<String>(Arrays.asList(SET_VALUES));
@Override
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 7b4ae42..6451173 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -576,14 +576,27 @@
BTree btree = component.getDeletedKeysBTree();
IIndexBulkLoader btreeBulkLoader = btree.createBulkLoader(1.0f, true, 0L, false);
+
+ long numElements = 0L;
+ for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
+ numElements += ((LSMInvertedIndexDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter().getNumElements();
+ }
+
+ int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
+ BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
+ bloomFilterFalsePositiveRate);
+ IIndexBulkLoader builder = component.getBloomFilter().createBuilder(numElements,
+ bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
try {
while (btreeCursor.hasNext()) {
btreeCursor.next();
ITupleReference tuple = btreeCursor.getTuple();
btreeBulkLoader.add(tuple);
+ builder.add(tuple);
}
} finally {
btreeCursor.close();
+ builder.end();
}
btreeBulkLoader.end();
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 95d1a11..2be6d57 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -46,7 +46,6 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackProvider;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessorInternal;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
@@ -322,14 +321,28 @@
BTree btree = mergedComponent.getBTree();
IIndexBulkLoader btreeBulkLoader = btree.createBulkLoader(1.0f, true, 0L, false);
+
+ long numElements = 0L;
+ for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
+ numElements += ((LSMRTreeDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter().getNumElements();
+ }
+
+ int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
+ BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
+ bloomFilterFalsePositiveRate);
+ IIndexBulkLoader builder = mergedComponent.getBloomFilter().createBuilder(numElements,
+ bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
+
try {
while (btreeCursor.hasNext()) {
btreeCursor.next();
ITupleReference tuple = btreeCursor.getTuple();
btreeBulkLoader.add(tuple);
+ builder.add(tuple);
}
} finally {
btreeCursor.close();
+ builder.end();
}
btreeBulkLoader.end();
}