Added LSM component-level filters for all indexes.
Change-Id: I898cf885c9f88feae85c99799a00fd8ec036efea
Reviewed-on: http://fulliautomatix.ics.uci.edu:8443/81
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
index b58c366..997053f 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
@@ -42,8 +42,9 @@
*/
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<S> dataSource,
List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
- IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context,
- JobSpecification jobSpec, Object implConfig) throws AlgebricksException;
+ List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema,
+ IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig)
+ throws AlgebricksException;
public boolean scannerOperatorIsLeaf(IDataSource<S> dataSource);
@@ -57,17 +58,18 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getWriteResultRuntime(IDataSource<S> dataSource,
IOperatorSchema propagatedSchema, List<LogicalVariable> keys, LogicalVariable payLoadVar,
- JobGenContext context, JobSpecification jobSpec) throws AlgebricksException;
+ List<LogicalVariable> additionalNonKeyFields, JobGenContext context, JobSpecification jobSpec)
+ throws AlgebricksException;
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<S> dataSource,
IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
- LogicalVariable payLoadVar, RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec)
- throws AlgebricksException;
+ LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, RecordDescriptor recordDesc,
+ JobGenContext context, JobSpecification jobSpec) throws AlgebricksException;
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(IDataSource<S> dataSource,
IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
- LogicalVariable payLoadVar, RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec)
- throws AlgebricksException;
+ LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, RecordDescriptor recordDesc,
+ JobGenContext context, JobSpecification jobSpec) throws AlgebricksException;
/**
* Creates the insert runtime of IndexInsertDeletePOperator, which models
@@ -85,6 +87,10 @@
* Variables for the dataset's primary keys that the dataSource secondary index belongs to.
* @param secondaryKeys
* Variables for the secondary-index keys.
+ * @param additionalNonKeyFields
+ * Additional variables that can be passed to the secondary index as payload.
+ * This can be useful when creating a second filter on a non-primary and non-secondary
+ * fields for additional pruning power.
* @param filterExpr
* Filtering expression to be pushed inside the runtime op.
* Such a filter may, e.g., exclude NULLs from being inserted/deleted.
@@ -101,8 +107,8 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexInsertRuntime(
IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
- ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec)
- throws AlgebricksException;
+ List<LogicalVariable> additionalNonKeyFields, ILogicalExpression filterExpr, RecordDescriptor recordDesc,
+ JobGenContext context, JobSpecification spec) throws AlgebricksException;
/**
* Creates the delete runtime of IndexInsertDeletePOperator, which models
@@ -120,6 +126,10 @@
* Variables for the dataset's primary keys that the dataSource secondary index belongs to.
* @param secondaryKeys
* Variables for the secondary-index keys.
+ * @param additionalNonKeyFields
+ * Additional variables that can be passed to the secondary index as payload.
+ * This can be useful when creating a second filter on a non-primary and non-secondary
+ * fields for additional pruning power.
* @param filterExpr
* Filtering expression to be pushed inside the runtime op.
* Such a filter may, e.g., exclude NULLs from being inserted/deleted.
@@ -136,8 +146,8 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexDeleteRuntime(
IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
- ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec)
- throws AlgebricksException;
+ List<LogicalVariable> additionalNonKeyFields, ILogicalExpression filterExpr, RecordDescriptor recordDesc,
+ JobGenContext context, JobSpecification spec) throws AlgebricksException;
public IDataSourceIndex<I, S> findDataSourceIndex(I indexId, S dataSourceId) throws AlgebricksException;
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
index f23b377..ba51f04 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
@@ -18,7 +18,10 @@
import java.util.Collection;
import java.util.List;
+import org.apache.commons.lang3.mutable.Mutable;
+
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -35,6 +38,10 @@
private boolean projectPushed = false;
+ private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
+ private List<LogicalVariable> minFilterVars;
+ private List<LogicalVariable> maxFilterVars;
+
public DataSourceScanOperator(List<LogicalVariable> variables, IDataSource<?> dataSource) {
super(variables);
this.dataSource = dataSource;
@@ -106,4 +113,28 @@
}
return env;
}
+
+ public List<LogicalVariable> getMinFilterVars() {
+ return minFilterVars;
+ }
+
+ public void setMinFilterVars(List<LogicalVariable> minFilterVars) {
+ this.minFilterVars = minFilterVars;
+ }
+
+ public List<LogicalVariable> getMaxFilterVars() {
+ return maxFilterVars;
+ }
+
+ public void setMaxFilterVars(List<LogicalVariable> maxFilterVars) {
+ this.maxFilterVars = maxFilterVars;
+ }
+
+ public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
+ this.additionalFilteringExpressions = additionalFilteringExpressions;
+ }
+
+ public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
+ return additionalFilteringExpressions;
+ }
}
\ No newline at end of file
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
index 587565f..ea45cc6 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
@@ -38,6 +38,7 @@
private final List<Mutable<ILogicalExpression>> secondaryKeyExprs;
private final Mutable<ILogicalExpression> filterExpr;
private final Kind operation;
+ private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
public IndexInsertDeleteOperator(IDataSourceIndex<?, ?> dataSourceIndex,
List<Mutable<ILogicalExpression>> primaryKeyExprs, List<Mutable<ILogicalExpression>> secondaryKeyExprs,
@@ -109,11 +110,19 @@
}
public Mutable<ILogicalExpression> getFilterExpression() {
- return filterExpr;
+ return filterExpr;
}
-
+
public Kind getOperation() {
return operation;
}
+ public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
+ this.additionalFilteringExpressions = additionalFilteringExpressions;
+ }
+
+ public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
+ return additionalFilteringExpressions;
+ }
+
}
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
index 4a558d1..94fd197 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
@@ -41,6 +41,7 @@
private final Mutable<ILogicalExpression> payloadExpr;
private final List<Mutable<ILogicalExpression>> primaryKeyExprs;
private final Kind operation;
+ private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
public InsertDeleteOperator(IDataSource<?> dataSource, Mutable<ILogicalExpression> payload,
List<Mutable<ILogicalExpression>> primaryKeyExprs, Kind operation) {
@@ -109,4 +110,12 @@
return operation;
}
+ public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
+ this.additionalFilteringExpressions = additionalFilteringExpressions;
+ }
+
+ public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
+ return additionalFilteringExpressions;
+ }
+
}
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
index f2d2f28..61e0b33 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
@@ -33,10 +33,12 @@
*/
public class UnnestMapOperator extends AbstractUnnestOperator {
- private final List<Object> variableTypes; // TODO: get rid of this and
- private final boolean propagateInput;
-
- // deprecate UnnestMap
+ private final List<Object> variableTypes; // TODO: get rid of this and deprecate UnnestMap
+ private boolean propagateInput;
+
+ private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
+ private List<LogicalVariable> minFilterVars;
+ private List<LogicalVariable> maxFilterVars;
public UnnestMapOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression,
List<Object> variableTypes, boolean propagateInput) {
@@ -94,11 +96,35 @@
}
return env;
}
-
+
public boolean propagatesInput() {
return propagateInput;
}
-
+
+ public List<LogicalVariable> getMinFilterVars() {
+ return minFilterVars;
+ }
+
+ public void setMinFilterVars(List<LogicalVariable> minFilterVars) {
+ this.minFilterVars = minFilterVars;
+ }
+
+ public List<LogicalVariable> getMaxFilterVars() {
+ return maxFilterVars;
+ }
+
+ public void setMaxFilterVars(List<LogicalVariable> maxFilterVars) {
+ this.maxFilterVars = maxFilterVars;
+ }
+
+ public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
+ this.additionalFilteringExpressions = additionalFilteringExpressions;
+ }
+
+ public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
+ return additionalFilteringExpressions;
+ }
+
/*
@Override
public boolean isMap() {
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
index f02d8e6..ba402bc 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
@@ -35,6 +35,7 @@
private IDataSource<?> dataSource;
private Mutable<ILogicalExpression> payloadExpr;
private List<Mutable<ILogicalExpression>> keyExprs;
+ private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
public WriteResultOperator(IDataSource<?> dataSource, Mutable<ILogicalExpression> payload,
List<Mutable<ILogicalExpression>> keyExprs) {
@@ -97,4 +98,12 @@
return createPropagatingAllInputsTypeEnvironment(ctx);
}
+ public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
+ this.additionalFilteringExpressions = additionalFilteringExpressions;
+ }
+
+ public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
+ return additionalFilteringExpressions;
+ }
+
}
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
index 9e94a5b..b56eb8c 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
@@ -771,16 +771,24 @@
public ILogicalOperator visitWriteResultOperator(WriteResultOperator op, Void arg) throws AlgebricksException {
ArrayList<Mutable<ILogicalExpression>> newKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
deepCopyExpressionRefs(newKeyExpressions, op.getKeyExpressions());
- return new WriteResultOperator(op.getDataSource(), deepCopyExpressionRef(op.getPayloadExpression()),
- newKeyExpressions);
+ List<Mutable<ILogicalExpression>> newLSMComponentFilterExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ deepCopyExpressionRefs(newKeyExpressions, op.getAdditionalFilteringExpressions());
+ WriteResultOperator writeResultOp = new WriteResultOperator(op.getDataSource(),
+ deepCopyExpressionRef(op.getPayloadExpression()), newKeyExpressions);
+ writeResultOp.setAdditionalFilteringExpressions(newLSMComponentFilterExpressions);
+ return writeResultOp;
}
@Override
public ILogicalOperator visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) throws AlgebricksException {
List<Mutable<ILogicalExpression>> newKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
deepCopyExpressionRefs(newKeyExpressions, op.getPrimaryKeyExpressions());
- return new InsertDeleteOperator(op.getDataSource(), deepCopyExpressionRef(op.getPayloadExpression()),
- newKeyExpressions, op.getOperation());
+ List<Mutable<ILogicalExpression>> newLSMComponentFilterExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ deepCopyExpressionRefs(newKeyExpressions, op.getAdditionalFilteringExpressions());
+ InsertDeleteOperator insertDeleteOp = new InsertDeleteOperator(op.getDataSource(),
+ deepCopyExpressionRef(op.getPayloadExpression()), newKeyExpressions, op.getOperation());
+ insertDeleteOp.setAdditionalFilteringExpressions(newLSMComponentFilterExpressions);
+ return insertDeleteOp;
}
@Override
@@ -792,8 +800,12 @@
deepCopyExpressionRefs(newSecondaryKeyExpressions, op.getSecondaryKeyExpressions());
Mutable<ILogicalExpression> newFilterExpression = new MutableObject<ILogicalExpression>(
((AbstractLogicalExpression) op.getFilterExpression()).cloneExpression());
- return new IndexInsertDeleteOperator(op.getDataSourceIndex(), newPrimaryKeyExpressions,
- newSecondaryKeyExpressions, newFilterExpression, op.getOperation());
+ List<Mutable<ILogicalExpression>> newLSMComponentFilterExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ deepCopyExpressionRefs(newLSMComponentFilterExpressions, op.getAdditionalFilteringExpressions());
+ IndexInsertDeleteOperator indexInsertDeleteOp = new IndexInsertDeleteOperator(op.getDataSourceIndex(),
+ newPrimaryKeyExpressions, newSecondaryKeyExpressions, newFilterExpression, op.getOperation());
+ indexInsertDeleteOp.setAdditionalFilteringExpressions(newLSMComponentFilterExpressions);
+ return indexInsertDeleteOp;
}
@Override
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
index 8dc545f..2bd9205 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
@@ -91,7 +91,11 @@
@Override
public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) {
- // does not use any variable
+ if (op.getAdditionalFilteringExpressions() != null) {
+ for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) {
+ e.getValue().getUsedVariables(usedVariables);
+ }
+ }
return null;
}
@@ -278,6 +282,11 @@
@Override
public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) {
op.getExpressionRef().getValue().getUsedVariables(usedVariables);
+ if (op.getAdditionalFilteringExpressions() != null) {
+ for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) {
+ e.getValue().getUsedVariables(usedVariables);
+ }
+ }
return null;
}
@@ -309,6 +318,11 @@
for (Mutable<ILogicalExpression> e : op.getKeyExpressions()) {
e.getValue().getUsedVariables(usedVariables);
}
+ if (op.getAdditionalFilteringExpressions() != null) {
+ for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) {
+ e.getValue().getUsedVariables(usedVariables);
+ }
+ }
return null;
}
@@ -318,6 +332,11 @@
for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
e.getValue().getUsedVariables(usedVariables);
}
+ if (op.getAdditionalFilteringExpressions() != null) {
+ for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) {
+ e.getValue().getUsedVariables(usedVariables);
+ }
+ }
return null;
}
@@ -329,6 +348,11 @@
for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
e.getValue().getUsedVariables(usedVariables);
}
+ if (op.getAdditionalFilteringExpressions() != null) {
+ for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) {
+ e.getValue().getUsedVariables(usedVariables);
+ }
+ }
return null;
}
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
index 98e4490..f1cb937 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
@@ -79,8 +79,10 @@
IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
List<LogicalVariable> vars = scan.getVariables();
List<LogicalVariable> projectVars = scan.getProjectVariables();
+
Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> p = mp.getScannerRuntime(dataSource, vars,
- projectVars, scan.isProjectPushed(), opSchema, typeEnv, context, builder.getJobSpec(), implConfig);
+ projectVars, scan.isProjectPushed(), scan.getMinFilterVars(), scan.getMaxFilterVars(), opSchema,
+ typeEnv, context, builder.getJobSpec(), implConfig);
builder.contributeHyracksOperator(scan, p.first);
if (p.second != null) {
builder.contributeAlgebricksPartitionConstraint(p.first, p.second);
@@ -89,5 +91,4 @@
ILogicalOperator srcExchange = scan.getInputs().get(0).getValue();
builder.contributeGraphEdge(srcExchange, 0, scan, 0);
}
-
}
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
index 879f2a9..0c07c4c 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
@@ -51,9 +51,11 @@
private final List<LogicalVariable> secondaryKeys;
private final ILogicalExpression filterExpr;
private final IDataSourceIndex<?, ?> dataSourceIndex;
+ private final List<LogicalVariable> additionalFilteringKeys;
public IndexInsertDeletePOperator(List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
- Mutable<ILogicalExpression> filterExpr, IDataSourceIndex<?, ?> dataSourceIndex) {
+ List<LogicalVariable> additionalFilteringKeys, Mutable<ILogicalExpression> filterExpr,
+ IDataSourceIndex<?, ?> dataSourceIndex) {
this.primaryKeys = primaryKeys;
this.secondaryKeys = secondaryKeys;
if (filterExpr != null) {
@@ -62,6 +64,7 @@
this.filterExpr = null;
}
this.dataSourceIndex = dataSourceIndex;
+ this.additionalFilteringKeys = additionalFilteringKeys;
}
@Override
@@ -105,10 +108,10 @@
IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(insertDeleteOp);
if (insertDeleteOp.getOperation() == Kind.INSERT) {
runtimeAndConstraints = mp.getIndexInsertRuntime(dataSourceIndex, propagatedSchema, inputSchemas, typeEnv,
- primaryKeys, secondaryKeys, filterExpr, inputDesc, context, spec);
+ primaryKeys, secondaryKeys, additionalFilteringKeys, filterExpr, inputDesc, context, spec);
} else {
runtimeAndConstraints = mp.getIndexDeleteRuntime(dataSourceIndex, propagatedSchema, inputSchemas, typeEnv,
- primaryKeys, secondaryKeys, filterExpr, inputDesc, context, spec);
+ primaryKeys, secondaryKeys, additionalFilteringKeys, filterExpr, inputDesc, context, spec);
}
builder.contributeHyracksOperator(insertDeleteOp, runtimeAndConstraints.first);
builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
index c85bfb9..5829911 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
@@ -48,11 +48,14 @@
private LogicalVariable payload;
private List<LogicalVariable> keys;
private IDataSource<?> dataSource;
+ private final List<LogicalVariable> additionalFilteringKeys;
- public InsertDeletePOperator(LogicalVariable payload, List<LogicalVariable> keys, IDataSource dataSource) {
+ public InsertDeletePOperator(LogicalVariable payload, List<LogicalVariable> keys,
+ List<LogicalVariable> additionalFilteringKeys, IDataSource dataSource) {
this.payload = payload;
this.keys = keys;
this.dataSource = dataSource;
+ this.additionalFilteringKeys = additionalFilteringKeys;
}
@Override
@@ -94,10 +97,10 @@
Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = null;
if (insertDeleteOp.getOperation() == Kind.INSERT) {
runtimeAndConstraints = mp.getInsertRuntime(dataSource, propagatedSchema, typeEnv, keys, payload,
- inputDesc, context, spec);
+ additionalFilteringKeys, inputDesc, context, spec);
} else {
runtimeAndConstraints = mp.getDeleteRuntime(dataSource, propagatedSchema, typeEnv, keys, payload,
- inputDesc, context, spec);
+ additionalFilteringKeys, inputDesc, context, spec);
}
builder.contributeHyracksOperator(insertDeleteOp, runtimeAndConstraints.first);
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
index 3b9d3d7..8b241d3 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
@@ -43,11 +43,14 @@
private LogicalVariable payload;
private List<LogicalVariable> keys;
private IDataSource<?> dataSource;
+ private final List<LogicalVariable> additionalFilteringKeys;
- public WriteResultPOperator(IDataSource<?> dataSource, LogicalVariable payload, List<LogicalVariable> keys) {
+ public WriteResultPOperator(IDataSource<?> dataSource, LogicalVariable payload, List<LogicalVariable> keys,
+ List<LogicalVariable> additionalFilteringKeys) {
this.dataSource = dataSource;
this.payload = payload;
this.keys = keys;
+ this.additionalFilteringKeys = additionalFilteringKeys;
}
@Override
@@ -93,7 +96,7 @@
JobSpecification spec = builder.getJobSpec();
Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getWriteResultRuntime(
- dataSource, propagatedSchema, keys, payload, context, spec);
+ dataSource, propagatedSchema, keys, payload, additionalFilteringKeys, context, spec);
builder.contributeHyracksOperator(writeResultOp, runtimeAndConstraints.first);
builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
index f441478..4b965f5 100644
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
@@ -78,8 +78,9 @@
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<String> dataSource,
List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
- IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context,
- JobSpecification jobSpec, Object implConfig) throws AlgebricksException {
+ List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema,
+ IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig)
+ throws AlgebricksException {
PigletFileDataSource ds = (PigletFileDataSource) dataSource;
FileSplit[] fileSplits = ds.getFileSplits();
@@ -165,7 +166,8 @@
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getWriteResultRuntime(
IDataSource<String> dataSource, IOperatorSchema propagatedSchema, List<LogicalVariable> keys,
- LogicalVariable payLoadVar, JobGenContext context, JobSpecification jobSpec) throws AlgebricksException {
+ LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, JobGenContext context,
+ JobSpecification jobSpec) throws AlgebricksException {
// TODO Auto-generated method stub
return null;
}
@@ -174,8 +176,9 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexInsertRuntime(
IDataSourceIndex<String, String> dataSource, IOperatorSchema propagatedSchema,
IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys,
- List<LogicalVariable> secondaryKeys, ILogicalExpression filterExpr, RecordDescriptor recordDesc,
- JobGenContext context, JobSpecification spec) throws AlgebricksException {
+ List<LogicalVariable> secondaryKeys, List<LogicalVariable> additionalNonKeyFields,
+ ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec)
+ throws AlgebricksException {
// TODO Auto-generated method stub
return null;
}
@@ -184,8 +187,9 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexDeleteRuntime(
IDataSourceIndex<String, String> dataSource, IOperatorSchema propagatedSchema,
IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys,
- List<LogicalVariable> secondaryKeys, ILogicalExpression filterExpr, RecordDescriptor recordDesc,
- JobGenContext context, JobSpecification spec) throws AlgebricksException {
+ List<LogicalVariable> secondaryKeys, List<LogicalVariable> additionalNonKeyFields,
+ ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec)
+ throws AlgebricksException {
// TODO Auto-generated method stub
return null;
}
@@ -198,8 +202,8 @@
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<String> dataSource,
IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
- LogicalVariable payLoadVar, RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec)
- throws AlgebricksException {
+ LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, RecordDescriptor recordDesc,
+ JobGenContext context, JobSpecification jobSpec) throws AlgebricksException {
// TODO Auto-generated method stub
return null;
}
@@ -207,8 +211,8 @@
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(IDataSource<String> dataSource,
IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
- LogicalVariable payLoadVar, RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec)
- throws AlgebricksException {
+ LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, RecordDescriptor recordDesc,
+ JobGenContext context, JobSpecification jobSpec) throws AlgebricksException {
// TODO Auto-generated method stub
return null;
}
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
index fd7d31d..58cb2ef 100644
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
+++ b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
@@ -268,26 +268,43 @@
WriteResultOperator opLoad = (WriteResultOperator) op;
LogicalVariable payload;
List<LogicalVariable> keys = new ArrayList<LogicalVariable>();
+ List<LogicalVariable> additionalFilteringKeys = null;
payload = getKeysAndLoad(opLoad.getPayloadExpression(), opLoad.getKeyExpressions(), keys);
- op.setPhysicalOperator(new WriteResultPOperator(opLoad.getDataSource(), payload, keys));
+ if (opLoad.getAdditionalFilteringExpressions() != null) {
+ additionalFilteringKeys = new ArrayList<LogicalVariable>();
+ getKeys(opLoad.getAdditionalFilteringExpressions(), additionalFilteringKeys);
+ }
+ op.setPhysicalOperator(new WriteResultPOperator(opLoad.getDataSource(), payload, keys,
+ additionalFilteringKeys));
break;
}
case INSERT_DELETE: {
InsertDeleteOperator opLoad = (InsertDeleteOperator) op;
LogicalVariable payload;
List<LogicalVariable> keys = new ArrayList<LogicalVariable>();
+ List<LogicalVariable> additionalFilteringKeys = null;
payload = getKeysAndLoad(opLoad.getPayloadExpression(), opLoad.getPrimaryKeyExpressions(), keys);
- op.setPhysicalOperator(new InsertDeletePOperator(payload, keys, opLoad.getDataSource()));
+ if (opLoad.getAdditionalFilteringExpressions() != null) {
+ additionalFilteringKeys = new ArrayList<LogicalVariable>();
+ getKeys(opLoad.getAdditionalFilteringExpressions(), additionalFilteringKeys);
+ }
+ op.setPhysicalOperator(new InsertDeletePOperator(payload, keys, additionalFilteringKeys, opLoad
+ .getDataSource()));
break;
}
case INDEX_INSERT_DELETE: {
IndexInsertDeleteOperator opInsDel = (IndexInsertDeleteOperator) op;
List<LogicalVariable> primaryKeys = new ArrayList<LogicalVariable>();
List<LogicalVariable> secondaryKeys = new ArrayList<LogicalVariable>();
+ List<LogicalVariable> additionalFilteringKeys = null;
getKeys(opInsDel.getPrimaryKeyExpressions(), primaryKeys);
getKeys(opInsDel.getSecondaryKeyExpressions(), secondaryKeys);
- op.setPhysicalOperator(new IndexInsertDeletePOperator(primaryKeys, secondaryKeys, opInsDel
- .getFilterExpression(), opInsDel.getDataSourceIndex()));
+ if (opInsDel.getAdditionalFilteringExpressions() != null) {
+ additionalFilteringKeys = new ArrayList<LogicalVariable>();
+ getKeys(opInsDel.getAdditionalFilteringExpressions(), additionalFilteringKeys);
+ }
+ op.setPhysicalOperator(new IndexInsertDeletePOperator(primaryKeys, secondaryKeys,
+ additionalFilteringKeys, opInsDel.getFilterExpression(), opInsDel.getDataSourceIndex()));
break;
}
case SINK: {
diff --git a/hivesterix/hivesterix-runtime/src/main/java/edu/uci/ics/hivesterix/runtime/jobgen/HiveMetaDataProvider.java b/hivesterix/hivesterix-runtime/src/main/java/edu/uci/ics/hivesterix/runtime/jobgen/HiveMetaDataProvider.java
index 65e6277..8c72a3e 100644
--- a/hivesterix/hivesterix-runtime/src/main/java/edu/uci/ics/hivesterix/runtime/jobgen/HiveMetaDataProvider.java
+++ b/hivesterix/hivesterix-runtime/src/main/java/edu/uci/ics/hivesterix/runtime/jobgen/HiveMetaDataProvider.java
@@ -12,139 +12,141 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.uci.ics.hivesterix.runtime.jobgen;
-
-import java.util.HashMap;
-import java.util.List;
-
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.plan.PartitionDesc;
-
-import edu.uci.ics.hivesterix.logical.expression.HiveFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-@SuppressWarnings("rawtypes")
-public class HiveMetaDataProvider<S, T> implements IMetadataProvider<S, T> {
-
- private Operator fileSink;
- private Schema outputSchema;
- private HashMap<S, IDataSource<S>> dataSourceMap;
-
- public HiveMetaDataProvider(Operator fsOp, Schema oi, HashMap<S, IDataSource<S>> map) {
- fileSink = fsOp;
- outputSchema = oi;
- dataSourceMap = map;
- }
-
- @Override
- public IDataSourceIndex<T, S> findDataSourceIndex(T indexId, S dataSourceId) throws AlgebricksException {
- return null;
- }
-
- @Override
- public IDataSource<S> findDataSource(S id) throws AlgebricksException {
- return dataSourceMap.get(id);
- }
-
- @Override
- public boolean scannerOperatorIsLeaf(IDataSource<S> dataSource) {
- return true;
- }
-
- @Override
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<S> dataSource,
- List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
- IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context,
- JobSpecification jobSpec, Object implConfig) throws AlgebricksException {
-
- S desc = dataSource.getId();
- HiveScanRuntimeGenerator generator = new HiveScanRuntimeGenerator((PartitionDesc) desc);
- return generator.getRuntimeOperatorAndConstraint(dataSource, scanVariables, projectVariables, projectPushed,
- context, jobSpec);
- }
-
- @Override
- public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink,
- int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc) {
-
- HiveWriteRuntimeGenerator generator = new HiveWriteRuntimeGenerator((FileSinkOperator) fileSink, outputSchema);
- return generator.getWriterRuntime(inputDesc);
- }
-
- @Override
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<S> dataSource,
- IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
- LogicalVariable payLoadVar, RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec)
- throws AlgebricksException {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(IDataSource<S> dataSource,
- IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
- LogicalVariable payLoadVar, RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec)
- throws AlgebricksException {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getResultHandleRuntime(IDataSink sink,
- int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc, boolean ordered,
- JobSpecification spec) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getWriteResultRuntime(IDataSource<S> arg0,
- IOperatorSchema arg1, List<LogicalVariable> arg2, LogicalVariable arg3, JobGenContext arg4,
- JobSpecification arg5) throws AlgebricksException {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public IFunctionInfo lookupFunction(FunctionIdentifier arg0) {
- return new HiveFunctionInfo(arg0, null);
- }
-
- @Override
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexInsertRuntime(
- IDataSourceIndex<T, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
- IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
- ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec)
- throws AlgebricksException {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexDeleteRuntime(
- IDataSourceIndex<T, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
- IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
- ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec)
- throws AlgebricksException {
- // TODO Auto-generated method stub
- return null;
- }
-}
+package edu.uci.ics.hivesterix.runtime.jobgen;
+
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+
+import edu.uci.ics.hivesterix.logical.expression.HiveFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+@SuppressWarnings("rawtypes")
+public class HiveMetaDataProvider<S, T> implements IMetadataProvider<S, T> {
+
+ private Operator fileSink;
+ private Schema outputSchema;
+ private HashMap<S, IDataSource<S>> dataSourceMap;
+
+ public HiveMetaDataProvider(Operator fsOp, Schema oi, HashMap<S, IDataSource<S>> map) {
+ fileSink = fsOp;
+ outputSchema = oi;
+ dataSourceMap = map;
+ }
+
+ @Override
+ public IDataSourceIndex<T, S> findDataSourceIndex(T indexId, S dataSourceId) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public IDataSource<S> findDataSource(S id) throws AlgebricksException {
+ return dataSourceMap.get(id);
+ }
+
+ @Override
+ public boolean scannerOperatorIsLeaf(IDataSource<S> dataSource) {
+ return true;
+ }
+
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<S> dataSource,
+ List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
+ List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema,
+ IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig)
+ throws AlgebricksException {
+
+ S desc = dataSource.getId();
+ HiveScanRuntimeGenerator generator = new HiveScanRuntimeGenerator((PartitionDesc) desc);
+ return generator.getRuntimeOperatorAndConstraint(dataSource, scanVariables, projectVariables, projectPushed,
+ context, jobSpec);
+ }
+
+ @Override
+ public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink,
+ int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc) {
+
+ HiveWriteRuntimeGenerator generator = new HiveWriteRuntimeGenerator((FileSinkOperator) fileSink, outputSchema);
+ return generator.getWriterRuntime(inputDesc);
+ }
+
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<S> dataSource,
+ IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
+ LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, RecordDescriptor recordDesc,
+ JobGenContext context, JobSpecification jobSpec) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(IDataSource<S> dataSource,
+ IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
+ LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, RecordDescriptor recordDesc,
+ JobGenContext context, JobSpecification jobSpec) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getResultHandleRuntime(IDataSink sink,
+ int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc, boolean ordered,
+ JobSpecification spec) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getWriteResultRuntime(IDataSource<S> arg0,
+ IOperatorSchema arg1, List<LogicalVariable> arg2, LogicalVariable arg3,
+ List<LogicalVariable> additionalNonKeyFields, JobGenContext arg4, JobSpecification arg5)
+ throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public IFunctionInfo lookupFunction(FunctionIdentifier arg0) {
+ return new HiveFunctionInfo(arg0, null);
+ }
+
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexInsertRuntime(
+ IDataSourceIndex<T, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
+ IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
+ List<LogicalVariable> additionalNonKeyFields, ILogicalExpression filterExpr, RecordDescriptor recordDesc,
+ JobGenContext context, JobSpecification spec) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexDeleteRuntime(
+ IDataSourceIndex<T, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
+ IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
+ List<LogicalVariable> additionalNonKeyFields, ILogicalExpression filterExpr, RecordDescriptor recordDesc,
+ JobGenContext context, JobSpecification spec) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+}
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
index a3d0772..9029b2e 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
@@ -63,7 +63,7 @@
@Option(name = "-btreename", usage = "B-Tree file name to search", required = true)
public String btreeName;
-
+
@Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
public int frameSize = 32768;
}
@@ -142,7 +142,9 @@
IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
BTreeSearchOperatorDescriptor btreeSearchOp = new BTreeSearchOperatorDescriptor(spec, recDesc, storageManager,
lcManagerProvider, btreeSplitProvider, typeTraits, comparatorFactories, null, lowKeyFields,
- highKeyFields, true, true, dataflowHelperFactory, false, false, null, NoOpOperationCallbackFactory.INSTANCE);
+ highKeyFields, true, true, dataflowHelperFactory, false, false, null,
+ NoOpOperationCallbackFactory.INSTANCE, null, null);
+
JobHelper.createPartitionConstraint(spec, btreeSearchOp, splitNCs);
// have each node print the results of its respective B-Tree
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
index 3af2e18..6d1f20e 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
@@ -66,7 +66,7 @@
@Option(name = "-secondary-btreename", usage = "Secondary B-Tree file name to search", required = true)
public String secondaryBTreeName;
-
+
@Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
public int frameSize = 32768;
}
@@ -169,7 +169,8 @@
BTreeSearchOperatorDescriptor secondarySearchOp = new BTreeSearchOperatorDescriptor(spec, secondaryRecDesc,
storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits,
searchComparatorFactories, null, secondaryLowKeyFields, secondaryHighKeyFields, true, true,
- dataflowHelperFactory, false, false, null, NoOpOperationCallbackFactory.INSTANCE);
+ dataflowHelperFactory, false, false, null, NoOpOperationCallbackFactory.INSTANCE, null, null);
+
JobHelper.createPartitionConstraint(spec, secondarySearchOp, splitNCs);
// secondary index will output tuples with [UTF8String, Integer]
@@ -184,8 +185,9 @@
IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName);
BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories,
- null, primaryLowKeyFields, primaryHighKeyFields, true, true, dataflowHelperFactory, false,
- false, null, NoOpOperationCallbackFactory.INSTANCE);
+ null, primaryLowKeyFields, primaryHighKeyFields, true, true, dataflowHelperFactory, false, false, null,
+ NoOpOperationCallbackFactory.INSTANCE, null, null);
+
JobHelper.createPartitionConstraint(spec, primarySearchOp, splitNCs);
// have each node print the results of its respective B-Tree
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/AbstractBTreeOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/AbstractBTreeOperatorTest.java
index 428d4fd..19b749a 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/AbstractBTreeOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/AbstractBTreeOperatorTest.java
@@ -228,7 +228,7 @@
BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories,
primaryBloomFilterKeyFields, lowKeyFields, highKeyFields, true, true, dataflowHelperFactory, false,
- false, null, NoOpOperationCallbackFactory.INSTANCE);
+ false, null, NoOpOperationCallbackFactory.INSTANCE, null, null);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
// sort based on secondary keys
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreePrimaryIndexScanOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreePrimaryIndexScanOperatorTest.java
index 76dd6f9..e4ea0fe 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreePrimaryIndexScanOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreePrimaryIndexScanOperatorTest.java
@@ -72,7 +72,7 @@
BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories,
primaryBloomFilterKeyFields, lowKeyFields, highKeyFields, true, true, dataflowHelperFactory, false,
- false, null, NoOpOperationCallbackFactory.INSTANCE);
+ false, null, NoOpOperationCallbackFactory.INSTANCE, null, null);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreePrimaryIndexSearchOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreePrimaryIndexSearchOperatorTest.java
index 2c41b39..c46bac7 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreePrimaryIndexSearchOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreePrimaryIndexSearchOperatorTest.java
@@ -77,7 +77,7 @@
BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories,
primaryBloomFilterKeyFields, lowKeyFields, highKeyFields, true, true, dataflowHelperFactory, false,
- false, null, NoOpOperationCallbackFactory.INSTANCE);
+ false, null, NoOpOperationCallbackFactory.INSTANCE, null, null);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeSecondaryIndexInsertOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeSecondaryIndexInsertOperatorTest.java
index 6962a82..d0a4e87 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeSecondaryIndexInsertOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeSecondaryIndexInsertOperatorTest.java
@@ -81,7 +81,9 @@
BTreeSearchOperatorDescriptor secondaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec,
secondaryRecDesc, storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits,
secondaryComparatorFactories, secondaryBloomFilterKeyFields, secondaryLowKeyFields,
- secondaryHighKeyFields, true, true, dataflowHelperFactory, false, false, null, NoOpOperationCallbackFactory.INSTANCE);
+ secondaryHighKeyFields, true, true, dataflowHelperFactory, false, false, null,
+ NoOpOperationCallbackFactory.INSTANCE, null, null);
+
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryBtreeSearchOp, NC1_ID);
// second field from the tuples coming from secondary index
@@ -93,7 +95,7 @@
BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories,
primaryBloomFilterKeyFields, primaryLowKeyFields, primaryHighKeyFields, true, true,
- dataflowHelperFactory, false, false, null, NoOpOperationCallbackFactory.INSTANCE);
+ dataflowHelperFactory, false, false, null, NoOpOperationCallbackFactory.INSTANCE, null, null);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeSecondaryIndexSearchOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeSecondaryIndexSearchOperatorTest.java
index 0341e74..4a1e690 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeSecondaryIndexSearchOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeSecondaryIndexSearchOperatorTest.java
@@ -80,7 +80,9 @@
BTreeSearchOperatorDescriptor secondaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec,
secondaryRecDesc, storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits,
secondaryComparatorFactories, secondaryBloomFilterKeyFields, secondaryLowKeyFields,
- secondaryHighKeyFields, true, true, dataflowHelperFactory, false, false, null, NoOpOperationCallbackFactory.INSTANCE);
+ secondaryHighKeyFields, true, true, dataflowHelperFactory, false, false, null,
+ NoOpOperationCallbackFactory.INSTANCE, null, null);
+
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryBtreeSearchOp, NC1_ID);
int[] primaryLowKeyFields = { 1 }; // second field from the tuples
@@ -92,7 +94,8 @@
BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories,
primaryBloomFilterKeyFields, primaryLowKeyFields, primaryHighKeyFields, true, true,
- dataflowHelperFactory, false, false, null, NoOpOperationCallbackFactory.INSTANCE);
+ dataflowHelperFactory, false, false, null, NoOpOperationCallbackFactory.INSTANCE, null, null);
+
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeSecondaryIndexUpsertOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeSecondaryIndexUpsertOperatorTest.java
index 6b503c1..b1371aa 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeSecondaryIndexUpsertOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeSecondaryIndexUpsertOperatorTest.java
@@ -80,7 +80,9 @@
BTreeSearchOperatorDescriptor secondaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec,
secondaryRecDesc, storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits,
secondaryComparatorFactories, secondaryBloomFilterKeyFields, secondaryLowKeyFields,
- secondaryHighKeyFields, true, true, dataflowHelperFactory, false, false, null, NoOpOperationCallbackFactory.INSTANCE);
+ secondaryHighKeyFields, true, true, dataflowHelperFactory, false, false, null,
+ NoOpOperationCallbackFactory.INSTANCE, null, null);
+
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryBtreeSearchOp, NC1_ID);
// second field from the tuples coming from secondary index
@@ -92,7 +94,7 @@
BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories,
primaryBloomFilterKeyFields, primaryLowKeyFields, primaryHighKeyFields, true, true,
- dataflowHelperFactory, false, false, null, NoOpOperationCallbackFactory.INSTANCE);
+ dataflowHelperFactory, false, false, null, NoOpOperationCallbackFactory.INSTANCE, null, null);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/AbstractfWordInvertedIndexTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/AbstractfWordInvertedIndexTest.java
index 30451e1..cfe6db4 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/AbstractfWordInvertedIndexTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/AbstractfWordInvertedIndexTest.java
@@ -193,8 +193,8 @@
}
private IOperatorDescriptor createFileScanOp(JobSpecification spec) {
- FileSplit[] dblpTitleFileSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data" + File.separator + "cleanednumbereddblptitles.txt"))) };
+ FileSplit[] dblpTitleFileSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File("data"
+ + File.separator + "cleanednumbereddblptitles.txt"))) };
IFileSplitProvider dblpTitleSplitProvider = new ConstantFileSplitProvider(dblpTitleFileSplits);
RecordDescriptor dblpTitleRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
@@ -237,7 +237,8 @@
BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
storageManager, lcManagerProvider, primaryFileSplitProvider, primaryTypeTraits,
primaryComparatorFactories, null, lowKeyFields, highKeyFields, true, true, btreeDataflowHelperFactory,
- false, false, null, NoOpOperationCallbackFactory.INSTANCE);
+ false, false, null, NoOpOperationCallbackFactory.INSTANCE, null, null);
+
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
return primaryBtreeSearchOp;
}
@@ -335,7 +336,8 @@
0, storageManager, btreeFileSplitProvider, lcManagerProvider, tokenTypeTraits,
tokenComparatorFactories, invListsTypeTraits, invListsComparatorFactories,
invertedIndexDataflowHelperFactory, tokenizerFactory, searchModifierFactory, invListsRecDesc, false,
- false, null, NoOpOperationCallbackFactory.INSTANCE);
+ false, null, NoOpOperationCallbackFactory.INSTANCE, null, null);
+
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, invIndexSearchOp, NC1_ID);
return invIndexSearchOp;
}
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java
index 8addae7..5d60f9d 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java
@@ -54,7 +54,8 @@
invertedIndexDataflowHelperFactory = new PartitionedLSMInvertedIndexDataflowHelperFactory(
virtualBufferCacheProvider, new ConstantMergePolicyFactory(), MERGE_POLICY_PROPERTIES,
ThreadCountingOperationTrackerProvider.INSTANCE, SynchronousSchedulerProvider.INSTANCE,
- NoOpIOOperationCallback.INSTANCE, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE);
+ NoOpIOOperationCallback.INSTANCE, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, null, null, null, null,
+ null, null);
}
@Override
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java
index 82de0c9..400a6b4 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java
@@ -49,7 +49,8 @@
invertedIndexDataflowHelperFactory = new LSMInvertedIndexDataflowHelperFactory(virtualBufferCacheProvider,
new ConstantMergePolicyFactory(), MERGE_POLICY_PROPERTIES,
ThreadCountingOperationTrackerProvider.INSTANCE, SynchronousSchedulerProvider.INSTANCE,
- NoOpIOOperationCallback.INSTANCE, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE);
+ NoOpIOOperationCallback.INSTANCE, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, null, null, null, null,
+ null, null);
}
@Override
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java
index f4d422b..849f650 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java
@@ -34,7 +34,7 @@
MERGE_POLICY_PROPERTIES = new HashMap<String, String>();
MERGE_POLICY_PROPERTIES.put("num-components", "3");
}
-
+
public LSMBTreeOperatorTestHelper(IOManager ioManager) {
super(ioManager);
}
@@ -43,7 +43,7 @@
return new LSMBTreeDataflowHelperFactory(virtualBufferCacheProvider, new ConstantMergePolicyFactory(),
MERGE_POLICY_PROPERTIES, ThreadCountingOperationTrackerProvider.INSTANCE,
SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE,
- DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, true);
+ DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, true, null, null, null, null);
}
}
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
index e848c7d..0799f99 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
@@ -50,7 +50,7 @@
return new LSMRTreeDataflowHelperFactory(valueProviderFactories, rtreePolicyType, btreeComparatorFactories,
virtualBufferCacheProvider, new ConstantMergePolicyFactory(), MERGE_POLICY_PROPERTIES,
ThreadCountingOperationTrackerProvider.INSTANCE, SynchronousSchedulerProvider.INSTANCE,
- NoOpIOOperationCallback.INSTANCE, linearizerCmpFactory, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE,
- btreeFields);
+ NoOpIOOperationCallback.INSTANCE, linearizerCmpFactory, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, null,
+ btreeFields, null, null, null);
}
}
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java
index fd6171a..df8b062 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java
@@ -49,6 +49,7 @@
return new LSMRTreeWithAntiMatterTuplesDataflowHelperFactory(valueProviderFactories, rtreePolicyType,
btreeComparatorFactories, virtualBufferCacheProvider, new ConstantMergePolicyFactory(),
MERGE_POLICY_PROPERTIES, ThreadCountingOperationTrackerProvider.INSTANCE,
- SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE, linearizerCmpFactory);
+ SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE, linearizerCmpFactory, null,
+ null, null, null);
}
}
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
index 6fc1d4b..2bdf652 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
@@ -188,15 +188,15 @@
rtreeDataflowHelperFactory = createDataFlowHelperFactory(secondaryValueProviderFactories,
RTreePolicyType.RSTARTREE, btreeComparatorFactories,
- LSMRTreeUtils.proposeBestLinearizer(secondaryTypeTraits, secondaryComparatorFactories.length), btreeFields);
+ LSMRTreeUtils.proposeBestLinearizer(secondaryTypeTraits, secondaryComparatorFactories.length),
+ btreeFields);
}
protected abstract IIndexDataflowHelperFactory createDataFlowHelperFactory(
IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType,
IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory,
- int[] btreeFields)
- throws TreeIndexException;
+ int[] btreeFields) throws TreeIndexException;
protected void createPrimaryIndex() throws Exception {
JobSpecification spec = new JobSpecification();
@@ -291,8 +291,8 @@
// scan primary index
BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories,
- null, lowKeyFields, highKeyFields, true, true, btreeDataflowHelperFactory, false,
- false, null, NoOpOperationCallbackFactory.INSTANCE);
+ null, lowKeyFields, highKeyFields, true, true, btreeDataflowHelperFactory, false, false, null,
+ NoOpOperationCallbackFactory.INSTANCE, null, null);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primarySearchOp, NC1_ID);
// load secondary index
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java
index 42cf8f0..7908d2b 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java
@@ -90,8 +90,9 @@
RTreeSearchOperatorDescriptor secondarySearchOp = new RTreeSearchOperatorDescriptor(spec, secondaryRecDesc,
storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits,
- secondaryComparatorFactories, keyFields, rtreeDataflowHelperFactory, false,
- false, null, NoOpOperationCallbackFactory.INSTANCE);
+ secondaryComparatorFactories, keyFields, rtreeDataflowHelperFactory, false, false, null,
+ NoOpOperationCallbackFactory.INSTANCE, null, null);
+
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondarySearchOp, NC1_ID);
// fifth field from the tuples coming from secondary index
@@ -102,8 +103,9 @@
// search primary index
BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories,
- null, primaryLowKeyFields, primaryHighKeyFields, true, true, btreeDataflowHelperFactory, false,
- false, null, NoOpOperationCallbackFactory.INSTANCE);
+ null, primaryLowKeyFields, primaryHighKeyFields, true, true, btreeDataflowHelperFactory, false, false,
+ null, NoOpOperationCallbackFactory.INSTANCE, null, null);
+
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primarySearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java
index cbd8b4c..ea88e20 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java
@@ -87,8 +87,8 @@
RTreeSearchOperatorDescriptor secondarySearchOp = new RTreeSearchOperatorDescriptor(spec, secondaryRecDesc,
storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits,
- secondaryComparatorFactories, keyFields, rtreeDataflowHelperFactory, false,
- false, null, NoOpOperationCallbackFactory.INSTANCE);
+ secondaryComparatorFactories, keyFields, rtreeDataflowHelperFactory, false, false, null,
+ NoOpOperationCallbackFactory.INSTANCE, null, null);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondarySearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java
index 2cabe52..432c2d7 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java
@@ -88,8 +88,8 @@
RTreeSearchOperatorDescriptor secondarySearchOp = new RTreeSearchOperatorDescriptor(spec, secondaryRecDesc,
storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits,
- secondaryComparatorFactories, keyFields, rtreeDataflowHelperFactory, false,
- false, null, NoOpOperationCallbackFactory.INSTANCE);
+ secondaryComparatorFactories, keyFields, rtreeDataflowHelperFactory, false, false, null,
+ NoOpOperationCallbackFactory.INSTANCE, null, null);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondarySearchOp, NC1_ID);
// fifth field from the tuples coming from secondary index
@@ -100,8 +100,8 @@
// search primary index
BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories,
- null, primaryLowKeyFields, primaryHighKeyFields, true, true, btreeDataflowHelperFactory, false,
- false, null, NoOpOperationCallbackFactory.INSTANCE);
+ null, primaryLowKeyFields, primaryHighKeyFields, true, true, btreeDataflowHelperFactory, false, false,
+ null, NoOpOperationCallbackFactory.INSTANCE, null, null);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primarySearchOp, NC1_ID);
IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
index 2f3f075..48f253c 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
@@ -40,6 +40,8 @@
protected final int[] highKeyFields;
protected final boolean lowKeyInclusive;
protected final boolean highKeyInclusive;
+ private final int[] minFilterFieldIndexes;
+ private final int[] maxFilterFieldIndexes;
public BTreeSearchOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc,
IStorageManagerInterface storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider,
@@ -47,7 +49,8 @@
IBinaryComparatorFactory[] comparatorFactories, int[] bloomFilterKeyFields, int[] lowKeyFields,
int[] highKeyFields, boolean lowKeyInclusive, boolean highKeyInclusive,
IIndexDataflowHelperFactory dataflowHelperFactory, boolean retainInput, boolean retainNull,
- INullWriterFactory nullWriterFactory, ISearchOperationCallbackFactory searchOpCallbackProvider) {
+ INullWriterFactory nullWriterFactory, ISearchOperationCallbackFactory searchOpCallbackProvider,
+ int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes) {
super(spec, 1, 1, recDesc, storageManager, lifecycleManagerProvider, fileSplitProvider, typeTraits,
comparatorFactories, bloomFilterKeyFields, dataflowHelperFactory, null, retainInput, retainNull,
nullWriterFactory, NoOpLocalResourceFactoryProvider.INSTANCE, searchOpCallbackProvider,
@@ -56,12 +59,14 @@
this.highKeyFields = highKeyFields;
this.lowKeyInclusive = lowKeyInclusive;
this.highKeyInclusive = highKeyInclusive;
+ this.minFilterFieldIndexes = minFilterFieldIndexes;
+ this.maxFilterFieldIndexes = maxFilterFieldIndexes;
}
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
return new BTreeSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, lowKeyFields,
- highKeyFields, lowKeyInclusive, highKeyInclusive);
+ highKeyFields, lowKeyInclusive, highKeyInclusive, minFilterFieldIndexes, maxFilterFieldIndexes);
}
}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
index 82e52c0..020adca 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
@@ -36,8 +36,8 @@
public BTreeSearchOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, IRecordDescriptorProvider recordDescProvider, int[] lowKeyFields, int[] highKeyFields,
- boolean lowKeyInclusive, boolean highKeyInclusive) {
- super(opDesc, ctx, partition, recordDescProvider);
+ boolean lowKeyInclusive, boolean highKeyInclusive, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes) {
+ super(opDesc, ctx, partition, recordDescProvider, minFilterFieldIndexes, maxFilterFieldIndexes);
this.lowKeyInclusive = lowKeyInclusive;
this.highKeyInclusive = highKeyInclusive;
if (lowKeyFields != null && lowKeyFields.length > 0) {
@@ -58,6 +58,12 @@
if (highKey != null) {
highKey.reset(accessor, tupleIndex);
}
+ if (minFilterKey != null) {
+ minFilterKey.reset(accessor, tupleIndex);
+ }
+ if (maxFilterKey != null) {
+ maxFilterKey.reset(accessor, tupleIndex);
+ }
}
@Override
@@ -65,7 +71,8 @@
ITreeIndex treeIndex = (ITreeIndex) index;
lowKeySearchCmp = BTreeUtils.getSearchMultiComparator(treeIndex.getComparatorFactories(), lowKey);
highKeySearchCmp = BTreeUtils.getSearchMultiComparator(treeIndex.getComparatorFactories(), highKey);
- return new RangePredicate(lowKey, highKey, lowKeyInclusive, highKeyInclusive, lowKeySearchCmp, highKeySearchCmp);
+ return new RangePredicate(lowKey, highKey, lowKeyInclusive, highKeyInclusive, lowKeySearchCmp,
+ highKeySearchCmp, minFilterKey, maxFilterKey);
}
@Override
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorDescriptor.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorDescriptor.java
index 195e177..afea036 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorDescriptor.java
@@ -44,7 +44,7 @@
ISearchOperationCallbackFactory searchOpCallbackProvider, ITupleUpdaterFactory tupleUpdaterFactory) {
super(spec, recDesc, storageManager, lifecycleManagerProvider, fileSplitProvider, typeTraits,
comparatorFactories, bloomFilterKeyFields, lowKeyFields, highKeyFields, lowKeyInclusive,
- highKeyInclusive, dataflowHelperFactory, retainInput, false, null, searchOpCallbackProvider);
+ highKeyInclusive, dataflowHelperFactory, retainInput, false, null, searchOpCallbackProvider, null, null);
this.tupleUpdaterFactory = tupleUpdaterFactory;
}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorNodePushable.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorNodePushable.java
index 198511e..e60026e 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorNodePushable.java
@@ -35,7 +35,7 @@
int partition, IRecordDescriptorProvider recordDescProvider, int[] lowKeyFields, int[] highKeyFields,
boolean lowKeyInclusive, boolean highKeyInclusive, ITupleUpdater tupleUpdater) {
super(opDesc, ctx, partition, recordDescProvider, lowKeyFields, highKeyFields, lowKeyInclusive,
- highKeyInclusive);
+ highKeyInclusive, null, null);
this.tupleUpdater = tupleUpdater;
}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/RangePredicate.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/RangePredicate.java
index 9924187..9038345 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/RangePredicate.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/RangePredicate.java
@@ -16,10 +16,10 @@
package edu.uci.ics.hyracks.storage.am.btree.impls;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.impls.AbstractSearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-public class RangePredicate implements ISearchPredicate {
+public class RangePredicate extends AbstractSearchPredicate {
private static final long serialVersionUID = 1L;
@@ -31,6 +31,7 @@
protected MultiComparator highKeyCmp;
public RangePredicate() {
+
}
public RangePredicate(ITupleReference lowKey, ITupleReference highKey, boolean lowKeyInclusive,
@@ -43,6 +44,18 @@
this.highKeyCmp = highKeyCmp;
}
+ public RangePredicate(ITupleReference lowKey, ITupleReference highKey, boolean lowKeyInclusive,
+ boolean highKeyInclusive, MultiComparator lowKeyCmp, MultiComparator highKeyCmp,
+ ITupleReference minFilterTuple, ITupleReference maxFilterTuple) {
+ super(minFilterTuple, maxFilterTuple);
+ this.lowKey = lowKey;
+ this.highKey = highKey;
+ this.lowKeyInclusive = lowKeyInclusive;
+ this.highKeyInclusive = highKeyInclusive;
+ this.lowKeyCmp = lowKeyCmp;
+ this.highKeyCmp = highKeyCmp;
+ }
+
public MultiComparator getLowKeyComparator() {
return lowKeyCmp;
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java
index e7a5ac7..a7278e8 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java
@@ -18,38 +18,44 @@
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
public interface ITreeIndexMetaDataFrame {
- public void initBuffer(byte level);
+ public void initBuffer(byte level);
- public void setPage(ICachedPage page);
+ public void setPage(ICachedPage page);
- public ICachedPage getPage();
+ public ICachedPage getPage();
- public byte getLevel();
+ public byte getLevel();
- public void setLevel(byte level);
+ public void setLevel(byte level);
- public int getNextPage();
+ public int getNextPage();
- public void setNextPage(int nextPage);
+ public void setNextPage(int nextPage);
- public int getMaxPage();
+ public int getMaxPage();
- public void setMaxPage(int maxPage);
+ public void setMaxPage(int maxPage);
- public int getFreePage();
+ public int getFreePage();
- public boolean hasSpace();
+ public boolean hasSpace();
- public void addFreePage(int freePage);
-
- // Special flag for LSM-Components to mark whether they are valid or not.
- public boolean isValid();
-
- // Set special validity flag.
- public void setValid(boolean isValid);
-
- // Special placeholder for LSN information. Used for transactional LSM indexes.
- public long getLSN();
-
- public void setLSN(long lsn);
+ public void addFreePage(int freePage);
+
+ // Special flag for LSM-Components to mark whether they are valid or not.
+ public boolean isValid();
+
+ // Set special validity flag.
+ public void setValid(boolean isValid);
+
+ // Return the lsm component filter page id.
+ public int getLSMComponentFilterPageId();
+
+ // Set the lsm component filter page id.
+ public void setLSMComponentFilterPageId(int filterPage);
+
+ // Special placeholder for LSN information. Used for transactional LSM indexes.
+ public long getLSN();
+
+ public void setLSN(long lsn);
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
index f2ff6eb..6d79e1f 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
@@ -37,6 +37,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.tuples.PermutingFrameTupleReference;
public abstract class IndexSearchOperatorNodePushable extends AbstractUnaryInputUnaryOutputOperatorNodePushable {
protected final IIndexOperatorDescriptor opDesc;
@@ -57,12 +58,18 @@
protected final RecordDescriptor inputRecDesc;
protected final boolean retainInput;
protected FrameTupleReference frameTuple;
+
protected final boolean retainNull;
protected ArrayTupleBuilder nullTupleBuild;
protected INullWriter nullWriter;
+ protected final int[] minFilterFieldIndexes;
+ protected final int[] maxFilterFieldIndexes;
+ protected PermutingFrameTupleReference minFilterKey;
+ protected PermutingFrameTupleReference maxFilterKey;
+
public IndexSearchOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- IRecordDescriptorProvider recordDescProvider) {
+ IRecordDescriptorProvider recordDescProvider, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes) {
this.opDesc = opDesc;
this.ctx = ctx;
this.indexHelper = opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(opDesc, ctx, partition);
@@ -72,6 +79,16 @@
this.nullWriter = opDesc.getNullWriterFactory().createNullWriter();
}
this.inputRecDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getActivityId(), 0);
+ this.minFilterFieldIndexes = minFilterFieldIndexes;
+ this.maxFilterFieldIndexes = maxFilterFieldIndexes;
+ if (minFilterFieldIndexes != null && minFilterFieldIndexes.length > 0) {
+ minFilterKey = new PermutingFrameTupleReference();
+ minFilterKey.setFieldPermutation(minFilterFieldIndexes);
+ }
+ if (maxFilterFieldIndexes != null && maxFilterFieldIndexes.length > 0) {
+ maxFilterKey = new PermutingFrameTupleReference();
+ maxFilterKey.setFieldPermutation(maxFilterFieldIndexes);
+ }
}
protected abstract ISearchPredicate createSearchPredicate();
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java
index 0a4d6c1..71e2b9f 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java
@@ -29,95 +29,101 @@
// Arbitrarily chosen magic integer.
protected static final int MAGIC_VALID_INT = 0x5bd1e995;
-
- protected static final int tupleCountOff = 0; //0
- protected static final int freeSpaceOff = tupleCountOff + 4; //4
- protected static final int maxPageOff = freeSpaceOff + 4; //8
- protected static final int levelOff = maxPageOff + 12; //20
- protected static final int nextPageOff = levelOff + 1; // 21
- protected static final int validOff = nextPageOff + 4; // 25
- protected static final int lsnOff = validOff + 4; // 29
- protected ICachedPage page = null;
- protected ByteBuffer buf = null;
+ protected static final int tupleCountOff = 0; //0
+ protected static final int freeSpaceOff = tupleCountOff + 4; //4
+ protected static final int maxPageOff = freeSpaceOff + 4; //8
+ protected static final int levelOff = maxPageOff + 12; //20
+ protected static final int nextPageOff = levelOff + 1; // 21
+ protected static final int validOff = nextPageOff + 4; // 25
- public int getMaxPage() {
- return buf.getInt(maxPageOff);
- }
+ // The additionalFilteringPageOff is used only for LSM indexes.
+ // We store the page id that will be used to store the information of the the filter that is associated with a disk component.
+ // It is only set in the first meta page other meta pages (i.e., with level -2) have junk in the max page field.
+ private static final int additionalFilteringPageOff = validOff + 4; // 29
+ protected static final int lsnOff = additionalFilteringPageOff + 4; // 33
- public void setMaxPage(int maxPage) {
- buf.putInt(maxPageOff, maxPage);
- }
+ protected ICachedPage page = null;
+ protected ByteBuffer buf = null;
- public int getFreePage() {
- int tupleCount = buf.getInt(tupleCountOff);
- if (tupleCount > 0) {
- // return the last page from the linked list of free pages
- // TODO: this is a dumb policy, but good enough for now
- int lastPageOff = buf.getInt(freeSpaceOff) - 4;
- buf.putInt(freeSpaceOff, lastPageOff);
- buf.putInt(tupleCountOff, tupleCount - 1);
- return buf.getInt(lastPageOff);
- } else {
- return -1;
- }
- }
+ public int getMaxPage() {
+ return buf.getInt(maxPageOff);
+ }
- // must be checked before adding free page
- // user of this class is responsible for getting a free page as a new meta
- // page, latching it, etc. if there is no space on this page
- public boolean hasSpace() {
- return buf.getInt(freeSpaceOff) + 4 < buf.capacity();
- }
+ public void setMaxPage(int maxPage) {
+ buf.putInt(maxPageOff, maxPage);
+ }
- // no bounds checking is done, there must be free space
- public void addFreePage(int freePage) {
- int freeSpace = buf.getInt(freeSpaceOff);
- buf.putInt(freeSpace, freePage);
- buf.putInt(freeSpaceOff, freeSpace + 4);
- buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
- }
+ public int getFreePage() {
+ int tupleCount = buf.getInt(tupleCountOff);
+ if (tupleCount > 0) {
+ // return the last page from the linked list of free pages
+ // TODO: this is a dumb policy, but good enough for now
+ int lastPageOff = buf.getInt(freeSpaceOff) - 4;
+ buf.putInt(freeSpaceOff, lastPageOff);
+ buf.putInt(tupleCountOff, tupleCount - 1);
+ return buf.getInt(lastPageOff);
+ } else {
+ return -1;
+ }
+ }
- @Override
- public byte getLevel() {
- return buf.get(levelOff);
- }
+ // must be checked before adding free page
+ // user of this class is responsible for getting a free page as a new meta
+ // page, latching it, etc. if there is no space on this page
+ public boolean hasSpace() {
+ return buf.getInt(freeSpaceOff) + 4 < buf.capacity();
+ }
- @Override
- public void setLevel(byte level) {
- buf.put(levelOff, level);
- }
+ // no bounds checking is done, there must be free space
+ public void addFreePage(int freePage) {
+ int freeSpace = buf.getInt(freeSpaceOff);
+ buf.putInt(freeSpace, freePage);
+ buf.putInt(freeSpaceOff, freeSpace + 4);
+ buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
+ }
- @Override
- public ICachedPage getPage() {
- return page;
- }
+ @Override
+ public byte getLevel() {
+ return buf.get(levelOff);
+ }
- @Override
- public void setPage(ICachedPage page) {
- this.page = page;
- this.buf = page.getBuffer();
- }
+ @Override
+ public void setLevel(byte level) {
+ buf.put(levelOff, level);
+ }
- @Override
- public void initBuffer(byte level) {
- buf.putInt(tupleCountOff, 0);
- buf.putInt(freeSpaceOff, lsnOff + 4);
- //buf.putInt(maxPageOff, -1);
- buf.put(levelOff, level);
- buf.putInt(nextPageOff, -1);
- setValid(false);
- }
+ @Override
+ public ICachedPage getPage() {
+ return page;
+ }
- @Override
- public int getNextPage() {
- return buf.getInt(nextPageOff);
- }
+ @Override
+ public void setPage(ICachedPage page) {
+ this.page = page;
+ this.buf = page.getBuffer();
+ }
- @Override
- public void setNextPage(int nextPage) {
- buf.putInt(nextPageOff, nextPage);
- }
+ @Override
+ public void initBuffer(byte level) {
+ buf.putInt(tupleCountOff, 0);
+ buf.putInt(freeSpaceOff, lsnOff + 8);
+ //buf.putInt(maxPageOff, -1);
+ buf.put(levelOff, level);
+ buf.putInt(nextPageOff, -1);
+ buf.putInt(additionalFilteringPageOff, -1);
+ setValid(false);
+ }
+
+ @Override
+ public int getNextPage() {
+ return buf.getInt(nextPageOff);
+ }
+
+ @Override
+ public void setNextPage(int nextPage) {
+ buf.putInt(nextPageOff, nextPage);
+ }
@Override
public boolean isValid() {
@@ -142,4 +148,14 @@
public void setLSN(long lsn) {
buf.putLong(lsnOff, lsn);
}
+
+ @Override
+ public int getLSMComponentFilterPageId() {
+ return buf.getInt(additionalFilteringPageOff);
+ }
+
+ @Override
+ public void setLSMComponentFilterPageId(int filterPage) {
+ buf.putInt(additionalFilteringPageOff, filterPage);
+ }
}
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/AbstractSearchPredicate.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/AbstractSearchPredicate.java
new file mode 100644
index 0000000..5a60b09
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/AbstractSearchPredicate.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.impls;
+
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+
+public abstract class AbstractSearchPredicate implements ISearchPredicate {
+
+ private static final long serialVersionUID = 1L;
+
+ protected ITupleReference minFilterTuple = null;
+ protected ITupleReference maxFilterTuple = null;
+
+ public AbstractSearchPredicate(ITupleReference minFilterTuple, ITupleReference maxFilterTuple) {
+ this.minFilterTuple = minFilterTuple;
+ this.maxFilterTuple = maxFilterTuple;
+ }
+
+ public AbstractSearchPredicate() {
+
+ }
+
+ public ITupleReference getMinFilterTuple() {
+ return minFilterTuple;
+ }
+
+ public ITupleReference getMaxFilterTuple() {
+ return maxFilterTuple;
+ }
+}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
index 2a458ac..983d893 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
@@ -38,7 +38,7 @@
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck, int version) {
super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory, ioScheduler,
- ioOpCallbackFactory, false);
+ ioOpCallbackFactory, false, null, null, null, null);
this.version = version;
}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
index 48dd41e..c44f18c 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
@@ -36,7 +36,7 @@
ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
double bloomFilterFalsePositiveRate, int version) {
super(null, mergePolicyFactory, mergePolicyProperties, opTrackerFactory, ioSchedulerProvider,
- ioOpCallbackFactory, bloomFilterFalsePositiveRate);
+ ioOpCallbackFactory, bloomFilterFalsePositiveRate, null, null, null);
this.version = version;
}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
index 7fb5009..768dd05 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
@@ -32,29 +32,28 @@
private int version;
public ExternalBTreeWithBuddyDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] buddyBtreeFields, int version) {
- super(opDesc, ctx, partition, null, mergePolicy, opTrackerFactory, ioScheduler,
- ioOpCallbackFactory);
+ int partition, ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ int[] buddyBtreeFields, int version) {
+ super(opDesc, ctx, partition, null, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, null,
+ null, null);
this.buddyBtreeFields = buddyBtreeFields;
this.version = version;
}
public ExternalBTreeWithBuddyDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition, double bloomFilterFalsePositiveRate,
- ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- int[] buddyBtreeFields, int version) {
- super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
- ioScheduler, ioOpCallbackFactory);
+ int partition, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
+ ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] buddyBtreeFields, int version) {
+ super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory, ioScheduler,
+ ioOpCallbackFactory, null, null, null);
this.buddyBtreeFields = buddyBtreeFields;
this.version = version;
}
-
+
@Override
public IIndex getIndexInstance() {
- if(index!= null)
+ if (index != null)
return index;
synchronized (lcManager) {
long resourceID;
@@ -75,14 +74,14 @@
@Override
protected IIndex createIndexInstance() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc;
- return LSMBTreeUtils.createExternalBTreeWithBuddy(file, opDesc.getStorageManager().getBufferCache(ctx),
- opDesc.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
- treeOpDesc.getTreeIndexComparatorFactories(), bloomFilterFalsePositiveRate, mergePolicy,
- opTrackerFactory.getOperationTracker(ctx), ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback(), buddyBtreeFields, version);
+ return LSMBTreeUtils.createExternalBTreeWithBuddy(file, opDesc.getStorageManager().getBufferCache(ctx), opDesc
+ .getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc
+ .getTreeIndexComparatorFactories(), bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory
+ .getOperationTracker(ctx), ioScheduler, ioOpCallbackFactory.createIOOperationCallback(),
+ buddyBtreeFields, version);
}
-
- public int getTargetVersion(){
+
+ public int getTargetVersion() {
return version;
}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
index 67f0c85..0aeb81d 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
@@ -31,13 +31,12 @@
private final int[] buddyBtreeFields;
private int version;
- public ExternalBTreeWithBuddyDataflowHelperFactory(
- ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
- ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, double bloomFilterFalsePositiveRate,
- int[] buddyBtreeFields, int version) {
- super(null, mergePolicyFactory, mergePolicyProperties, opTrackerFactory,
- ioSchedulerProvider, ioOpCallbackFactory, bloomFilterFalsePositiveRate);
+ public ExternalBTreeWithBuddyDataflowHelperFactory(ILSMMergePolicyFactory mergePolicyFactory,
+ Map<String, String> mergePolicyProperties, ILSMOperationTrackerProvider opTrackerFactory,
+ ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ double bloomFilterFalsePositiveRate, int[] buddyBtreeFields, int version) {
+ super(null, mergePolicyFactory, mergePolicyProperties, opTrackerFactory, ioSchedulerProvider,
+ ioOpCallbackFactory, bloomFilterFalsePositiveRate, null, null, null);
this.buddyBtreeFields = buddyBtreeFields;
this.version = version;
}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
index 2708a81..b4bd166 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
@@ -18,6 +18,8 @@
import java.util.List;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
@@ -31,24 +33,31 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.AbstractLSMIndexDataflowHelper;
public class LSMBTreeDataflowHelper extends AbstractLSMIndexDataflowHelper {
-
+
private final boolean needKeyDupCheck;
+ private final int[] btreeFields;
public LSMBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
List<IVirtualBufferCache> virtualBufferCaches, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields,
+ int[] filterFields) {
this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
- opTrackerFactory, ioScheduler, ioOpCallbackFactory, needKeyDupCheck);
+ opTrackerFactory, ioScheduler, ioOpCallbackFactory, needKeyDupCheck, filterTypeTraits,
+ filterCmpFactories, btreeFields, filterFields);
}
public LSMBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
List<IVirtualBufferCache> virtualBufferCaches, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck) {
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
+ int[] btreeFields, int[] filterFields) {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
- ioScheduler, ioOpCallbackFactory);
+ ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields);
this.needKeyDupCheck = needKeyDupCheck;
+ this.btreeFields = btreeFields;
}
@Override
@@ -58,6 +67,7 @@
opDesc.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
treeOpDesc.getTreeIndexComparatorFactories(), treeOpDesc.getTreeIndexBloomFilterKeyFields(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx), ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback(), needKeyDupCheck);
+ ioOpCallbackFactory.createIOOperationCallback(), needKeyDupCheck, filterTypeTraits, filterCmpFactories,
+ btreeFields, filterFields);
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
index 7fe7702..01b4e5f 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
@@ -18,6 +18,8 @@
import java.util.Map;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -31,16 +33,19 @@
private static final long serialVersionUID = 1L;
private final boolean needKeyDupCheck;
+ private final int[] btreeFields;
public LSMBTreeDataflowHelperFactory(IVirtualBufferCacheProvider virtualBufferCacheProvider,
ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, double bloomFilterFalsePositiveRate,
- boolean needKeyDupCheck) {
+ boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
+ int[] btreeFields, int[] filterFields) {
super(virtualBufferCacheProvider, mergePolicyFactory, mergePolicyProperties, opTrackerFactory,
- ioSchedulerProvider, ioOpCallbackFactory, bloomFilterFalsePositiveRate);
+ ioSchedulerProvider, ioOpCallbackFactory, bloomFilterFalsePositiveRate, filterTypeTraits,
+ filterCmpFactories, filterFields);
this.needKeyDupCheck = needKeyDupCheck;
-
+ this.btreeFields = btreeFields;
}
@Override
@@ -49,6 +54,7 @@
return new LSMBTreeDataflowHelper(opDesc, ctx, partition,
virtualBufferCacheProvider.getVirtualBufferCaches(ctx), bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties), opTrackerFactory,
- ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, needKeyDupCheck);
+ ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, needKeyDupCheck, filterTypeTraits,
+ filterCmpFactories, btreeFields, filterFields);
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index da19882..b417152 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -94,7 +94,7 @@
super(interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, fileManager, diskBTreeFactory,
bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider,
fieldCount, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, false);
- this.transactionComponentFactory = new LSMBTreeDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory);
+ this.transactionComponentFactory = new LSMBTreeDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory, null);
this.secondDiskComponents = new LinkedList<ILSMComponent>();
this.interiorFrameFactory = interiorFrameFactory;
this.version = version;
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeOpContext.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeOpContext.java
index 220943f..68ec7d1 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeOpContext.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeOpContext.java
@@ -21,6 +21,7 @@
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
@@ -28,94 +29,99 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
public class ExternalBTreeOpContext implements ILSMIndexOperationContext {
- public ITreeIndexFrameFactory insertLeafFrameFactory;
- public ITreeIndexFrameFactory deleteLeafFrameFactory;
- public IBTreeLeafFrame insertLeafFrame;
- public IBTreeLeafFrame deleteLeafFrame;
- public IndexOperation op;
- public final MultiComparator cmp;
- public final MultiComparator bloomFilterCmp;
- public final ISearchOperationCallback searchCallback;
- private final List<ILSMComponent> componentHolder;
- private final List<ILSMComponent> componentsToBeMerged;
- private final int targetIndexVersion;
+ public ITreeIndexFrameFactory insertLeafFrameFactory;
+ public ITreeIndexFrameFactory deleteLeafFrameFactory;
+ public IBTreeLeafFrame insertLeafFrame;
+ public IBTreeLeafFrame deleteLeafFrame;
+ public IndexOperation op;
+ public final MultiComparator cmp;
+ public final MultiComparator bloomFilterCmp;
+ public final ISearchOperationCallback searchCallback;
+ private final List<ILSMComponent> componentHolder;
+ private final List<ILSMComponent> componentsToBeMerged;
+ private final int targetIndexVersion;
+ public ISearchPredicate searchPredicate;
- public ExternalBTreeOpContext(
- ITreeIndexFrameFactory insertLeafFrameFactory,
- ITreeIndexFrameFactory deleteLeafFrameFactory,
- ISearchOperationCallback searchCallback,
- int numBloomFilterKeyFields,
- IBinaryComparatorFactory[] cmpFactories, int targetIndexVersion) {
- if (cmpFactories != null) {
- this.cmp = MultiComparator.create(cmpFactories);
- } else {
- this.cmp = null;
- }
- bloomFilterCmp = MultiComparator.create(cmpFactories, 0,
- numBloomFilterKeyFields);
- this.insertLeafFrameFactory = insertLeafFrameFactory;
- this.deleteLeafFrameFactory = deleteLeafFrameFactory;
- this.insertLeafFrame = (IBTreeLeafFrame) insertLeafFrameFactory
- .createFrame();
- this.deleteLeafFrame = (IBTreeLeafFrame) deleteLeafFrameFactory
- .createFrame();
- if (insertLeafFrame != null && this.cmp != null) {
- insertLeafFrame.setMultiComparator(cmp);
- }
- if (deleteLeafFrame != null && this.cmp != null) {
- deleteLeafFrame.setMultiComparator(cmp);
- }
- this.componentHolder = new LinkedList<ILSMComponent>();
- this.componentsToBeMerged = new LinkedList<ILSMComponent>();
- this.searchCallback = searchCallback;
- this.targetIndexVersion = targetIndexVersion;
- }
+ public ExternalBTreeOpContext(ITreeIndexFrameFactory insertLeafFrameFactory,
+ ITreeIndexFrameFactory deleteLeafFrameFactory, ISearchOperationCallback searchCallback,
+ int numBloomFilterKeyFields, IBinaryComparatorFactory[] cmpFactories, int targetIndexVersion) {
+ if (cmpFactories != null) {
+ this.cmp = MultiComparator.create(cmpFactories);
+ } else {
+ this.cmp = null;
+ }
+ bloomFilterCmp = MultiComparator.create(cmpFactories, 0, numBloomFilterKeyFields);
+ this.insertLeafFrameFactory = insertLeafFrameFactory;
+ this.deleteLeafFrameFactory = deleteLeafFrameFactory;
+ this.insertLeafFrame = (IBTreeLeafFrame) insertLeafFrameFactory.createFrame();
+ this.deleteLeafFrame = (IBTreeLeafFrame) deleteLeafFrameFactory.createFrame();
+ if (insertLeafFrame != null && this.cmp != null) {
+ insertLeafFrame.setMultiComparator(cmp);
+ }
+ if (deleteLeafFrame != null && this.cmp != null) {
+ deleteLeafFrame.setMultiComparator(cmp);
+ }
+ this.componentHolder = new LinkedList<ILSMComponent>();
+ this.componentsToBeMerged = new LinkedList<ILSMComponent>();
+ this.searchCallback = searchCallback;
+ this.targetIndexVersion = targetIndexVersion;
+ }
- @Override
- public void setOperation(IndexOperation newOp) {
- reset();
- this.op = newOp;
- }
+ @Override
+ public void setOperation(IndexOperation newOp) {
+ reset();
+ this.op = newOp;
+ }
- @Override
- public void reset() {
- componentHolder.clear();
- componentsToBeMerged.clear();
- }
+ @Override
+ public void reset() {
+ componentHolder.clear();
+ componentsToBeMerged.clear();
+ }
- public IndexOperation getOperation() {
- return op;
- }
+ public IndexOperation getOperation() {
+ return op;
+ }
- @Override
- public List<ILSMComponent> getComponentHolder() {
- return componentHolder;
- }
+ @Override
+ public List<ILSMComponent> getComponentHolder() {
+ return componentHolder;
+ }
- @Override
- public ISearchOperationCallback getSearchOperationCallback() {
- return searchCallback;
- }
+ @Override
+ public ISearchOperationCallback getSearchOperationCallback() {
+ return searchCallback;
+ }
- // Disk only index should never needs a modification callback
- @Override
- public IModificationOperationCallback getModificationCallback() {
- return null;
- }
+ // Disk only index should never needs a modification callback
+ @Override
+ public IModificationOperationCallback getModificationCallback() {
+ return null;
+ }
- @Override
- public void setCurrentMutableComponentId(int currentMutableComponentId) {
- // Do nothing: this method should never be called for this class
- }
+ @Override
+ public void setCurrentMutableComponentId(int currentMutableComponentId) {
+ // Do nothing: this method should never be called for this class
+ }
- @Override
- public List<ILSMComponent> getComponentsToBeMerged() {
- return componentsToBeMerged;
- }
+ @Override
+ public List<ILSMComponent> getComponentsToBeMerged() {
+ return componentsToBeMerged;
+ }
- // Used by indexes with global transaction
- public int getTargetIndexVersion() {
- return targetIndexVersion;
- }
+ // Used by indexes with global transaction
+ public int getTargetIndexVersion() {
+ return targetIndexVersion;
+ }
+
+ @Override
+ public void setSearchPredicate(ISearchPredicate searchPredicate) {
+ this.searchPredicate = searchPredicate;
+ }
+
+ @Override
+ public ISearchPredicate getSearchPredicate() {
+ return searchPredicate;
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddyOpContext.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddyOpContext.java
index e676bc8..645810a 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddyOpContext.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddyOpContext.java
@@ -20,85 +20,95 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
-public class ExternalBTreeWithBuddyOpContext implements
- ILSMIndexOperationContext {
- private IndexOperation op;
- private MultiComparator bTreeCmp;
- private MultiComparator buddyBTreeCmp;
- public final List<ILSMComponent> componentHolder;
- private final List<ILSMComponent> componentsToBeMerged;
- public final ISearchOperationCallback searchCallback;
- private final int targetIndexVersion;
+public class ExternalBTreeWithBuddyOpContext implements ILSMIndexOperationContext {
+ private IndexOperation op;
+ private MultiComparator bTreeCmp;
+ private MultiComparator buddyBTreeCmp;
+ public final List<ILSMComponent> componentHolder;
+ private final List<ILSMComponent> componentsToBeMerged;
+ public final ISearchOperationCallback searchCallback;
+ private final int targetIndexVersion;
+ public ISearchPredicate searchPredicate;
- public ExternalBTreeWithBuddyOpContext(
- IBinaryComparatorFactory[] btreeCmpFactories,
- IBinaryComparatorFactory[] buddyBtreeCmpFactories,
- ISearchOperationCallback searchCallback, int targetIndexVersion) {
+ public ExternalBTreeWithBuddyOpContext(IBinaryComparatorFactory[] btreeCmpFactories,
+ IBinaryComparatorFactory[] buddyBtreeCmpFactories, ISearchOperationCallback searchCallback,
+ int targetIndexVersion) {
- this.componentHolder = new LinkedList<ILSMComponent>();
- this.componentsToBeMerged = new LinkedList<ILSMComponent>();
- this.searchCallback = searchCallback;
- this.targetIndexVersion = targetIndexVersion;
- this.bTreeCmp = MultiComparator.create(btreeCmpFactories);
- this.buddyBTreeCmp = MultiComparator.create(buddyBtreeCmpFactories);
- }
+ this.componentHolder = new LinkedList<ILSMComponent>();
+ this.componentsToBeMerged = new LinkedList<ILSMComponent>();
+ this.searchCallback = searchCallback;
+ this.targetIndexVersion = targetIndexVersion;
+ this.bTreeCmp = MultiComparator.create(btreeCmpFactories);
+ this.buddyBTreeCmp = MultiComparator.create(buddyBtreeCmpFactories);
+ }
- public void setOperation(IndexOperation newOp) {
- reset();
- this.op = newOp;
- }
+ public void setOperation(IndexOperation newOp) {
+ reset();
+ this.op = newOp;
+ }
- @Override
- public void setCurrentMutableComponentId(int currentMutableComponentId) {
- // Do nothing. this should never be called for disk only indexes
- }
+ @Override
+ public void setCurrentMutableComponentId(int currentMutableComponentId) {
+ // Do nothing. this should never be called for disk only indexes
+ }
- @Override
- public void reset() {
- componentHolder.clear();
- componentsToBeMerged.clear();
- }
+ @Override
+ public void reset() {
+ componentHolder.clear();
+ componentsToBeMerged.clear();
+ }
- @Override
- public IndexOperation getOperation() {
- return op;
- }
+ @Override
+ public IndexOperation getOperation() {
+ return op;
+ }
- public MultiComparator getBTreeMultiComparator() {
- return bTreeCmp;
- }
+ public MultiComparator getBTreeMultiComparator() {
+ return bTreeCmp;
+ }
- public MultiComparator getBuddyBTreeMultiComparator() {
- return buddyBTreeCmp;
- }
+ public MultiComparator getBuddyBTreeMultiComparator() {
+ return buddyBTreeCmp;
+ }
- @Override
- public List<ILSMComponent> getComponentHolder() {
- return componentHolder;
- }
+ @Override
+ public List<ILSMComponent> getComponentHolder() {
+ return componentHolder;
+ }
- @Override
- public ISearchOperationCallback getSearchOperationCallback() {
- return searchCallback;
- }
+ @Override
+ public ISearchOperationCallback getSearchOperationCallback() {
+ return searchCallback;
+ }
- // This should never be needed for disk only indexes
- @Override
- public IModificationOperationCallback getModificationCallback() {
- return null;
- }
+ // This should never be needed for disk only indexes
+ @Override
+ public IModificationOperationCallback getModificationCallback() {
+ return null;
+ }
- @Override
- public List<ILSMComponent> getComponentsToBeMerged() {
- return componentsToBeMerged;
- }
+ @Override
+ public List<ILSMComponent> getComponentsToBeMerged() {
+ return componentsToBeMerged;
+ }
- public int getTargetIndexVersion() {
- return targetIndexVersion;
- }
+ public int getTargetIndexVersion() {
+ return targetIndexVersion;
+ }
+
+ @Override
+ public void setSearchPredicate(ISearchPredicate searchPredicate) {
+ this.searchPredicate = searchPredicate;
+ }
+
+ @Override
+ public ISearchPredicate getSearchPredicate() {
+ return searchPredicate;
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 9a0e315..4c29ff2 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -16,6 +16,7 @@
package edu.uci.ics.hyracks.storage.am.lsm.btree.impls;
import java.io.File;
+import java.util.ArrayList;
import java.util.List;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -46,11 +47,15 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.exceptions.TreeIndexDuplicateKeyException;
+import edu.uci.ics.hyracks.storage.am.common.impls.AbstractSearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.tuples.PermutingTupleReference;
import edu.uci.ics.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleReference;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
@@ -66,6 +71,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
@@ -85,24 +91,29 @@
protected final ITreeIndexFrameFactory deleteLeafFrameFactory;
protected final IBinaryComparatorFactory[] cmpFactories;
- protected final boolean needKeyDupCheck;
+ private final boolean needKeyDupCheck;
+ private final int[] btreeFields;
public LSMBTree(List<IVirtualBufferCache> virtualBufferCaches, ITreeIndexFrameFactory interiorFrameFactory,
ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory,
ILSMIndexFileManager fileManager, TreeIndexFactory<BTree> diskBTreeFactory,
TreeIndexFactory<BTree> bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory,
- double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount,
- IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck) {
+ ILSMComponentFilterFactory filterFactory, ILSMComponentFilterFrameFactory filterFrameFactory,
+ LSMComponentFilterManager filterManager, double bloomFilterFalsePositiveRate,
+ IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, int[] btreeFields, int[] filterFields) {
super(virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
- bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback);
+ bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
+ filterManager, filterFields);
int i = 0;
for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
LSMBTreeMemoryComponent mutableComponent = new LSMBTreeMemoryComponent(new BTree(virtualBufferCache,
virtualBufferCache.getFileMapProvider(), new VirtualFreePageManager(
virtualBufferCache.getNumPages()), interiorFrameFactory, insertLeafFrameFactory,
cmpFactories, fieldCount, new FileReference(new File(fileManager.getBaseDir() + "_virtual_" + i))),
- virtualBufferCache, i == 0 ? true : false);
+ virtualBufferCache, i == 0 ? true : false, filterFactory == null ? null
+ : filterFactory.createLSMComponentFilter());
memoryComponents.add(mutableComponent);
++i;
}
@@ -110,9 +121,11 @@
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
this.cmpFactories = cmpFactories;
- componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory);
- bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory);
+ componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, filterFactory);
+ bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory,
+ filterFactory);
this.needKeyDupCheck = needKeyDupCheck;
+ this.btreeFields = btreeFields;
}
// Without memory components
@@ -128,9 +141,10 @@
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
this.cmpFactories = cmpFactories;
- componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory);
- bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory);
+ componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, null);
+ bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory, null);
this.needKeyDupCheck = needKeyDupCheck;
+ this.btreeFields = null;
}
@Override
@@ -273,9 +287,7 @@
case DELETE:
operationalComponents.add(memoryComponents.get(cmc));
break;
- case SEARCH:
case INSERT:
-
for (int i = 0; i < numMutableComponents - 1; i++) {
ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
@@ -288,6 +300,31 @@
operationalComponents.add(0, memoryComponents.get(cmc));
operationalComponents.addAll(immutableComponents);
break;
+ case SEARCH:
+ for (int i = 0; i < numMutableComponents - 1; i++) {
+ ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+ if (mutableComponent.isReadable()) {
+ // Make sure newest components are added first
+ operationalComponents.add(0, mutableComponent);
+ }
+ }
+ // The current mutable component is always added
+ operationalComponents.add(0, memoryComponents.get(cmc));
+ if (filterManager != null) {
+ for (ILSMComponent c : immutableComponents) {
+ if (c.getLSMComponentFilter().satisfy(
+ ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMinFilterTuple(),
+ ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMaxFilterTuple(),
+ ((LSMBTreeOpContext) ctx).filterCmp)) {
+ operationalComponents.add(c);
+ }
+ }
+ } else {
+ operationalComponents.addAll(immutableComponents);
+ }
+
+ break;
case MERGE:
operationalComponents.addAll(ctx.getComponentsToBeMerged());
break;
@@ -302,17 +339,31 @@
@Override
public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
LSMBTreeOpContext ctx = (LSMBTreeOpContext) ictx;
+
+ ITupleReference indexTuple;
+ if (ctx.indexTuple != null) {
+ ctx.indexTuple.reset(tuple);
+ indexTuple = ctx.indexTuple;
+ } else {
+ indexTuple = tuple;
+ }
+
switch (ctx.getOperation()) {
case PHYSICALDELETE:
- ctx.currentMutableBTreeAccessor.delete(tuple);
+ ctx.currentMutableBTreeAccessor.delete(indexTuple);
break;
case INSERT:
- insert(tuple, ctx);
+ insert(indexTuple, ctx);
break;
default:
- ctx.currentMutableBTreeAccessor.upsert(tuple);
+ ctx.currentMutableBTreeAccessor.upsert(indexTuple);
break;
}
+ if (ctx.filterTuple != null) {
+ ctx.filterTuple.reset(tuple);
+ memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter()
+ .update(ctx.filterTuple, ctx.filterCmp);
+ }
}
private boolean insert(ITupleReference tuple, LSMBTreeOpContext ctx) throws HyracksDataException, IndexException {
@@ -440,6 +491,14 @@
builder.end();
}
bulkLoader.end();
+
+ if (component.getLSMComponentFilter() != null) {
+ List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
+ filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
+ filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
+ filterManager.writeFilterInfo(component.getLSMComponentFilter(), component.getBTree());
+ }
return component;
}
@@ -504,6 +563,17 @@
builder.end();
}
bulkLoader.end();
+
+ if (mergedComponent.getLSMComponentFilter() != null) {
+ List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
+ filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
+ filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
+ }
+ filterManager.updateFilterInfo(mergedComponent.getLSMComponentFilter(), filterTuples);
+ filterManager.writeFilterInfo(mergedComponent.getLSMComponentFilter(), mergedComponent.getBTree());
+ }
+
return mergedComponent;
}
@@ -520,6 +590,9 @@
// BTree will be closed during cleanup of merge().
component.getBTree().activate();
component.getBloomFilter().activate();
+ if (component.getLSMComponentFilter() != null) {
+ filterManager.readFilterInfo(component.getLSMComponentFilter(), component.getBTree());
+ }
return component;
}
@@ -561,6 +634,9 @@
private boolean cleanedUpArtifacts = false;
private boolean isEmptyComponent = true;
private boolean endedBloomFilterLoad = false;
+ public final PermutingTupleReference indexTuple;
+ public final PermutingTupleReference filterTuple;
+ public final MultiComparator filterCmp;
public LSMBTreeBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex)
throws TreeIndexException, HyracksDataException {
@@ -580,13 +656,36 @@
bloomFilterFalsePositiveRate);
builder = ((LSMBTreeDiskComponent) component).getBloomFilter().createBuilder(numElementsHint,
bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
+
+ if (filterFields != null) {
+ indexTuple = new PermutingTupleReference(btreeFields);
+ filterCmp = MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories());
+ filterTuple = new PermutingTupleReference(filterFields);
+ } else {
+ indexTuple = null;
+ filterCmp = null;
+ filterTuple = null;
+ }
}
@Override
public void add(ITupleReference tuple) throws IndexException, HyracksDataException {
try {
- bulkLoader.add(tuple);
- builder.add(tuple);
+ ITupleReference t;
+ if (indexTuple != null) {
+ indexTuple.reset(tuple);
+ t = indexTuple;
+ } else {
+ t = tuple;
+ }
+
+ bulkLoader.add(t);
+ builder.add(t);
+
+ if (filterTuple != null) {
+ filterTuple.reset(tuple);
+ component.getLSMComponentFilter().update(filterTuple, filterCmp);
+ }
} catch (IndexException | HyracksDataException | RuntimeException e) {
cleanupArtifacts();
throw e;
@@ -619,6 +718,12 @@
endedBloomFilterLoad = true;
}
bulkLoader.end();
+
+ if (component.getLSMComponentFilter() != null) {
+ filterManager.writeFilterInfo(component.getLSMComponentFilter(),
+ ((LSMBTreeDiskComponent) component).getBTree());
+ }
+
if (isEmptyComponent) {
cleanupArtifacts();
} else {
@@ -631,7 +736,8 @@
public LSMBTreeOpContext createOpContext(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
return new LSMBTreeOpContext(memoryComponents, insertLeafFrameFactory, deleteLeafFrameFactory,
- modificationCallback, searchCallback, componentFactory.getBloomFilterKeyFields().length);
+ modificationCallback, searchCallback, componentFactory.getBloomFilterKeyFields().length, btreeFields,
+ filterFields);
}
@Override
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
index 381b012..7346d9c 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
@@ -17,13 +17,15 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractDiskLSMComponent;
public class LSMBTreeDiskComponent extends AbstractDiskLSMComponent {
private final BTree btree;
private final BloomFilter bloomFilter;
- public LSMBTreeDiskComponent(BTree btree, BloomFilter bloomFilter) {
+ public LSMBTreeDiskComponent(BTree btree, BloomFilter bloomFilter, ILSMComponentFilter filter) {
+ super(filter);
this.btree = btree;
this.bloomFilter = bloomFilter;
}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java
index 7b1e0de..275781c 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java
@@ -21,6 +21,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
@@ -28,17 +29,21 @@
public class LSMBTreeDiskComponentFactory implements ILSMComponentFactory {
private final TreeIndexFactory<BTree> btreeFactory;
private final BloomFilterFactory bloomFilterFactory;
+ private final ILSMComponentFilterFactory filterFactory;
- public LSMBTreeDiskComponentFactory(TreeIndexFactory<BTree> btreeFactory, BloomFilterFactory bloomFilterFactory) {
+ public LSMBTreeDiskComponentFactory(TreeIndexFactory<BTree> btreeFactory, BloomFilterFactory bloomFilterFactory,
+ ILSMComponentFilterFactory filterFactory) {
this.btreeFactory = btreeFactory;
this.bloomFilterFactory = bloomFilterFactory;
+ this.filterFactory = filterFactory;
}
@Override
public ILSMComponent createLSMComponentInstance(LSMComponentFileReferences cfr) throws IndexException,
HyracksDataException {
return new LSMBTreeDiskComponent(btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
- bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()));
+ bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()),
+ filterFactory == null ? null : filterFactory.createLSMComponentFilter());
}
@Override
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java
index 7550349..7b4633e 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java
@@ -17,6 +17,7 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractMemoryLSMComponent;
@@ -24,8 +25,8 @@
private final BTree btree;
- public LSMBTreeMemoryComponent(BTree btree, IVirtualBufferCache vbc, boolean isActive) {
- super(vbc, isActive);
+ public LSMBTreeMemoryComponent(BTree btree, IVirtualBufferCache vbc, boolean isActive, ILSMComponentFilter filter) {
+ super(vbc, isActive, filter);
this.btree = btree;
}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
index cb7bae7..a2598ea 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
@@ -24,10 +24,12 @@
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.tuples.PermutingTupleReference;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
@@ -49,10 +51,14 @@
public final ISearchOperationCallback searchCallback;
private final List<ILSMComponent> componentHolder;
private final List<ILSMComponent> componentsToBeMerged;
+ public final PermutingTupleReference indexTuple;
+ public final MultiComparator filterCmp;
+ public final PermutingTupleReference filterTuple;
+ public ISearchPredicate searchPredicate;
public LSMBTreeOpContext(List<ILSMComponent> mutableComponents, ITreeIndexFrameFactory insertLeafFrameFactory,
ITreeIndexFrameFactory deleteLeafFrameFactory, IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback, int numBloomFilterKeyFields) {
+ ISearchOperationCallback searchCallback, int numBloomFilterKeyFields, int[] btreeFields, int[] filterFields) {
LSMBTreeMemoryComponent c = (LSMBTreeMemoryComponent) mutableComponents.get(0);
IBinaryComparatorFactory cmpFactories[] = c.getBTree().getComparatorFactories();
if (cmpFactories[0] != null) {
@@ -88,6 +94,16 @@
this.componentsToBeMerged = new LinkedList<ILSMComponent>();
this.modificationCallback = modificationCallback;
this.searchCallback = searchCallback;
+
+ if (filterFields != null) {
+ indexTuple = new PermutingTupleReference(btreeFields);
+ filterCmp = MultiComparator.create(c.getLSMComponentFilter().getFilterCmpFactories());
+ filterTuple = new PermutingTupleReference(filterFields);
+ } else {
+ indexTuple = null;
+ filterCmp = null;
+ filterTuple = null;
+ }
}
@Override
@@ -161,4 +177,14 @@
public List<ILSMComponent> getComponentsToBeMerged() {
return componentsToBeMerged;
}
+
+ @Override
+ public void setSearchPredicate(ISearchPredicate searchPredicate) {
+ this.searchPredicate = searchPredicate;
+ }
+
+ @Override
+ public ISearchPredicate getSearchPredicate() {
+ return searchPredicate;
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
index 6d7564b..3523017 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
@@ -44,7 +44,10 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.common.frames.LSMComponentFilterFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFilterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@@ -54,7 +57,8 @@
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck) {
+ ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields) {
LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
cmpFactories.length, false);
LSMBTreeTupleWriterFactory deleteTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
@@ -78,12 +82,24 @@
BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider,
bloomFilterKeyFields);
+ LSMComponentFilterFactory filterFactory = null;
+ LSMComponentFilterFrameFactory filterFrameFactory = null;
+ LSMComponentFilterManager filterManager = null;
+ if (filterCmpFactories != null) {
+ TypeAwareTupleWriterFactory filterTupleWriterFactory = new TypeAwareTupleWriterFactory(filterTypeTraits);
+ filterFactory = new LSMComponentFilterFactory(filterTupleWriterFactory, filterCmpFactories);
+ filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory,
+ diskBufferCache.getPageSize());
+ filterManager = new LSMComponentFilterManager(diskBufferCache, filterFrameFactory);
+ }
+
ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(diskFileMapProvider, file, diskBTreeFactory);
LSMBTree lsmTree = new LSMBTree(virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
- bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, cmpFactories, mergePolicy,
- opTracker, ioScheduler, ioOpCallback, needKeyDupCheck);
+ filterFactory, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskFileMapProvider,
+ typeTraits.length, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck,
+ btreeFields, filterFields);
return lsmTree;
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponent.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponent.java
index afbabe0..5927b17 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponent.java
@@ -41,4 +41,6 @@
public LSMComponentType getType();
public ComponentState getState();
+
+ public ILSMComponentFilter getLSMComponentFilter();
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFilter.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFilter.java
new file mode 100644
index 0000000..9039ab5
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFilter.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.common.api;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+public interface ILSMComponentFilter {
+
+ public void update(ITupleReference tuple, MultiComparator cmp);
+
+ public boolean satisfy(ITupleReference minTuple, ITupleReference maxTuple, MultiComparator filterCmp);
+
+ public ITupleReference getMinTuple();
+
+ public ITupleReference getMaxTuple();
+
+ public IBinaryComparatorFactory[] getFilterCmpFactories();
+
+ public void reset();
+}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFactory.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFactory.java
new file mode 100644
index 0000000..8cca485
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFactory.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.common.api;
+
+public interface ILSMComponentFilterFactory {
+ public ILSMComponentFilter createLSMComponentFilter();
+
+}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrame.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrame.java
new file mode 100644
index 0000000..05db79d
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrame.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.common.api;
+
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+
+public interface ILSMComponentFilterFrame {
+
+ public ICachedPage getPage();
+
+ public void setPage(ICachedPage page);
+
+ public void writeMinTuple(ITupleReference tuple);
+
+ public void writeMaxTuple(ITupleReference tuple);
+
+ public void initBuffer();
+
+ public boolean isMinTupleSet();
+
+ public boolean isMaxTupleSet();
+
+ public ITupleReference getMinTuple();
+
+ public ITupleReference getMaxTuple();
+
+}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrameFactory.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrameFactory.java
new file mode 100644
index 0000000..68ba177
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFilterFrameFactory.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.common.api;
+
+public interface ILSMComponentFilterFrameFactory {
+ public ILSMComponentFilterFrame createFrame();
+}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFilterManager.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFilterManager.java
new file mode 100644
index 0000000..59b349b
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMComponentFilterManager.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.common.api;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+
+public interface ILSMComponentFilterManager {
+
+ public void updateFilterInfo(ILSMComponentFilter filter, List<ITupleReference> filterTuples)
+ throws HyracksDataException;
+
+ public boolean readFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex) throws HyracksDataException;
+
+ public void writeFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex) throws HyracksDataException;
+
+}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java
index 80264bc..954ae1b 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexOperationContext.java
@@ -19,10 +19,11 @@
import edu.uci.ics.hyracks.storage.am.common.api.IIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
public interface ILSMIndexOperationContext extends IIndexOperationContext {
public List<ILSMComponent> getComponentHolder();
-
+
public List<ILSMComponent> getComponentsToBeMerged();
public ISearchOperationCallback getSearchOperationCallback();
@@ -30,4 +31,8 @@
public IModificationOperationCallback getModificationCallback();
public void setCurrentMutableComponentId(int currentMutableComponentId);
+
+ public void setSearchPredicate(ISearchPredicate searchPredicate);
+
+ public ISearchPredicate getSearchPredicate();
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java
index e277edc..8900385 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java
@@ -18,6 +18,8 @@
import java.util.List;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -37,19 +39,24 @@
protected final ILSMIOOperationScheduler ioScheduler;
protected final ILSMOperationTrackerProvider opTrackerFactory;
protected final ILSMIOOperationCallbackFactory ioOpCallbackFactory;
+ protected final ITypeTraits[] filterTypeTraits;
+ protected final IBinaryComparatorFactory[] filterCmpFactories;
+ protected final int[] filterFields;
public AbstractLSMIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
List<IVirtualBufferCache> virtualBufferCaches, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) {
this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
- opTrackerFactory, ioScheduler, ioOpCallbackFactory);
+ opTrackerFactory, ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields);
}
public AbstractLSMIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
List<IVirtualBufferCache> virtualBufferCaches, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory) {
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) {
super(opDesc, ctx, partition);
this.virtualBufferCaches = virtualBufferCaches;
this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
@@ -57,5 +64,8 @@
this.opTrackerFactory = opTrackerFactory;
this.ioScheduler = ioScheduler;
this.ioOpCallbackFactory = ioOpCallbackFactory;
+ this.filterTypeTraits = filterTypeTraits;
+ this.filterCmpFactories = filterCmpFactories;
+ this.filterFields = filterFields;
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelperFactory.java
index 4276ba7..9d034aa 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelperFactory.java
@@ -17,6 +17,8 @@
import java.util.Map;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
@@ -34,11 +36,15 @@
protected final ILSMIOOperationSchedulerProvider ioSchedulerProvider;
protected final ILSMIOOperationCallbackFactory ioOpCallbackFactory;
protected final double bloomFilterFalsePositiveRate;
+ protected final ITypeTraits[] filterTypeTraits;
+ protected final IBinaryComparatorFactory[] filterCmpFactories;
+ protected final int[] filterFields;
public AbstractLSMIndexDataflowHelperFactory(IVirtualBufferCacheProvider virtualBufferCacheProvider,
ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, double bloomFilterFalsePositiveRate) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, double bloomFilterFalsePositiveRate,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) {
this.virtualBufferCacheProvider = virtualBufferCacheProvider;
this.mergePolicyFactory = mergePolicyFactory;
this.opTrackerFactory = opTrackerFactory;
@@ -46,5 +52,8 @@
this.ioOpCallbackFactory = ioOpCallbackFactory;
this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
this.mergePolicyProperties = mergePolicyProperties;
+ this.filterTypeTraits = filterTypeTraits;
+ this.filterCmpFactories = filterCmpFactories;
+ this.filterFields = filterFields;
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrame.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrame.java
new file mode 100644
index 0000000..e2b5584
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrame.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.common.frames;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrame;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+
+public class LSMComponentFilterFrame implements ILSMComponentFilterFrame {
+
+ // This page consists of two tuples that represents the minimum and maximum tuples in an LSM component.
+
+ // A-one byte to indicate whether the filter tuples were set yet.
+ private static final int minTupleIsSetIndicatorOff = 0;
+ private static final int maxTupleIsSetIndicatorOff = 1;
+
+ private final int minTupleOff;
+ private final int maxTupleOff;
+
+ private final ITreeIndexTupleWriter tupleWriter;
+
+ protected ICachedPage page = null;
+ protected ByteBuffer buf = null;
+
+ private ITreeIndexTupleReference minTuple;
+ private ITreeIndexTupleReference maxTuple;
+
+ public LSMComponentFilterFrame(ITreeIndexTupleWriter tupleWriter, int pageSize) {
+ this.tupleWriter = tupleWriter;
+ this.minTupleOff = maxTupleIsSetIndicatorOff + 1;
+ this.maxTupleOff = maxTupleIsSetIndicatorOff + 1 + (pageSize / 2);
+
+ this.minTuple = tupleWriter.createTupleReference();
+ this.maxTuple = tupleWriter.createTupleReference();
+ }
+
+ @Override
+ public void initBuffer() {
+ buf.put(minTupleIsSetIndicatorOff, (byte) 0);
+ buf.put(maxTupleIsSetIndicatorOff, (byte) 0);
+ }
+
+ @Override
+ public ICachedPage getPage() {
+ return page;
+ }
+
+ @Override
+ public void setPage(ICachedPage page) {
+ this.page = page;
+ this.buf = page.getBuffer();
+ }
+
+ @Override
+ public void writeMinTuple(ITupleReference tuple) {
+ tupleWriter.writeTuple(tuple, buf.array(), minTupleOff);
+ buf.put(minTupleIsSetIndicatorOff, (byte) 1);
+ }
+
+ @Override
+ public void writeMaxTuple(ITupleReference tuple) {
+ tupleWriter.writeTuple(tuple, buf.array(), maxTupleOff);
+ buf.put(maxTupleIsSetIndicatorOff, (byte) 1);
+ }
+
+ @Override
+ public boolean isMinTupleSet() {
+ return buf.get(minTupleIsSetIndicatorOff) == (byte) 1 ? true : false;
+ }
+
+ @Override
+ public boolean isMaxTupleSet() {
+ return buf.get(maxTupleIsSetIndicatorOff) == (byte) 1 ? true : false;
+ }
+
+ @Override
+ public ITupleReference getMinTuple() {
+ minTuple.resetByTupleOffset(buf, minTupleOff);
+ return minTuple;
+ }
+
+ @Override
+ public ITupleReference getMaxTuple() {
+ maxTuple.resetByTupleOffset(buf, maxTupleOff);
+ return maxTuple;
+ }
+}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrameFactory.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrameFactory.java
new file mode 100644
index 0000000..41cec97
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/frames/LSMComponentFilterFrameFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.common.frames;
+
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrame;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
+
+public class LSMComponentFilterFrameFactory implements ILSMComponentFilterFrameFactory {
+
+ private final ITreeIndexTupleWriterFactory tupleWriterFactory;
+ private final int pageSize;
+
+ public LSMComponentFilterFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory, int pageSize) {
+ this.tupleWriterFactory = tupleWriterFactory;
+ this.pageSize = pageSize;
+ }
+
+ @Override
+ public ILSMComponentFilterFrame createFrame() {
+ return new LSMComponentFilterFrame(tupleWriterFactory.createTupleWriter(), pageSize);
+ }
+}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractDiskLSMComponent.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractDiskLSMComponent.java
index ec12c23..5765f71 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractDiskLSMComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractDiskLSMComponent.java
@@ -15,14 +15,19 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
public abstract class AbstractDiskLSMComponent extends AbstractLSMComponent {
- public AbstractDiskLSMComponent() {
- super();
+ public AbstractDiskLSMComponent(ILSMComponentFilter filter) {
+ super(filter);
state = ComponentState.READABLE_UNWRITABLE;
}
+ public AbstractDiskLSMComponent() {
+ this(null);
+ }
+
@Override
public boolean threadEnter(LSMOperationType opType, boolean isMutableComponent) {
assert state != ComponentState.INACTIVE;
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMComponent.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMComponent.java
index 5dab25f..e2c31aa 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMComponent.java
@@ -15,18 +15,30 @@
package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
public abstract class AbstractLSMComponent implements ILSMComponent {
protected ComponentState state;
protected int readerCount;
+ protected final ILSMComponentFilter filter;
- public AbstractLSMComponent() {
+ public AbstractLSMComponent(ILSMComponentFilter filter) {
+ this.filter = filter;
readerCount = 0;
}
+
+ public AbstractLSMComponent() {
+ this(null);
+ }
@Override
public ComponentState getState() {
return state;
}
+
+ @Override
+ public ILSMComponentFilter getLSMComponentFilter() {
+ return filter;
+ }
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index e5fcf0b..e8014a7 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -25,6 +25,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -55,6 +56,9 @@
protected final IFileMapProvider diskFileMapProvider;
protected final List<ILSMComponent> diskComponents;
protected final double bloomFilterFalsePositiveRate;
+ protected final ILSMComponentFilterFrameFactory filterFrameFactory;
+ protected final LSMComponentFilterManager filterManager;
+ protected final int[] filterFields;
protected boolean isActivated;
@@ -63,7 +67,9 @@
public AbstractLSMIndex(List<IVirtualBufferCache> virtualBufferCaches, IBufferCache diskBufferCache,
ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback) {
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
+ int[] filterFields) {
this.virtualBufferCaches = virtualBufferCaches;
this.diskBufferCache = diskBufferCache;
this.diskFileMapProvider = diskFileMapProvider;
@@ -72,6 +78,9 @@
this.ioScheduler = ioScheduler;
this.ioOpCallback = ioOpCallback;
this.ioOpCallback.setNumOfMutableComponents(virtualBufferCaches.size());
+ this.filterFrameFactory = filterFrameFactory;
+ this.filterManager = filterManager;
+ this.filterFields = filterFields;
lsmHarness = new LSMHarness(this, mergePolicy, opTracker);
isActivated = false;
diskComponents = new LinkedList<ILSMComponent>();
@@ -101,6 +110,9 @@
memoryComponents = null;
currentMutableComponentId = null;
flushRequests = null;
+ filterFrameFactory = null;
+ filterManager = null;
+ filterFields = null;
}
protected void forceFlushDirtyPages(ITreeIndex treeIndex) throws HyracksDataException {
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMemoryLSMComponent.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMemoryLSMComponent.java
index ce4817b..3907bc1 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMemoryLSMComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractMemoryLSMComponent.java
@@ -17,6 +17,7 @@
import java.util.concurrent.atomic.AtomicBoolean;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
public abstract class AbstractMemoryLSMComponent extends AbstractLSMComponent {
@@ -26,8 +27,8 @@
private final AtomicBoolean isModified;
private boolean requestedToBeActive;
- public AbstractMemoryLSMComponent(IVirtualBufferCache vbc, boolean isActive) {
- super();
+ public AbstractMemoryLSMComponent(IVirtualBufferCache vbc, boolean isActive, ILSMComponentFilter filter) {
+ super(filter);
this.vbc = vbc;
writerCount = 0;
if (isActive) {
@@ -38,6 +39,10 @@
isModified = new AtomicBoolean();
}
+ public AbstractMemoryLSMComponent(IVirtualBufferCache vbc, boolean isActive) {
+ this(vbc, isActive, null);
+ }
+
@Override
public boolean threadEnter(LSMOperationType opType, boolean isMutableComponent) throws HyracksDataException {
if (state == ComponentState.INACTIVE && requestedToBeActive) {
@@ -174,5 +179,8 @@
protected void reset() throws HyracksDataException {
isModified.set(false);
+ if (filter != null) {
+ filter.reset();
+ }
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMComponentFilter.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMComponentFilter.java
new file mode 100644
index 0000000..8cb66c1
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMComponentFilter.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
+
+public class LSMComponentFilter implements ILSMComponentFilter {
+
+ private final IBinaryComparatorFactory[] filterCmpFactories;
+ private final ITreeIndexTupleWriter tupleWriter;
+
+ private ITupleReference minTuple;
+ private ITupleReference maxTuple;
+
+ private byte[] minTupleBytes;
+ private ByteBuffer minTupleBuf;
+
+ private byte[] maxTupleBytes;
+ private ByteBuffer maxTupleBuf;
+
+ public LSMComponentFilter(ITreeIndexTupleWriter tupleWriter, IBinaryComparatorFactory[] filterCmpFactories) {
+ this.filterCmpFactories = filterCmpFactories;
+ this.tupleWriter = tupleWriter;
+ }
+
+ @Override
+ public IBinaryComparatorFactory[] getFilterCmpFactories() {
+ return filterCmpFactories;
+ }
+
+ @Override
+ public void reset() {
+ minTuple = null;
+ maxTuple = null;
+ minTupleBytes = null;
+ maxTupleBytes = null;
+ minTupleBuf = null;
+ maxTupleBuf = null;
+ }
+
+ @Override
+ public void update(ITupleReference tuple, MultiComparator cmp) {
+ if (minTuple == null) {
+ int numBytes = tupleWriter.bytesRequired(tuple);
+ minTupleBytes = new byte[numBytes];
+ tupleWriter.writeTuple(tuple, minTupleBytes, 0);
+ minTupleBuf = ByteBuffer.wrap(minTupleBytes);
+ minTuple = tupleWriter.createTupleReference();
+ ((ITreeIndexTupleReference) minTuple).resetByTupleOffset(minTupleBuf, 0);
+ } else {
+ int c = cmp.compare(tuple, minTuple);
+ if (c < 0) {
+ int numBytes = tupleWriter.bytesRequired(tuple);
+ if (minTupleBytes.length < numBytes) {
+ minTupleBytes = new byte[numBytes];
+ tupleWriter.writeTuple(tuple, minTupleBytes, 0);
+ minTupleBuf = ByteBuffer.wrap(minTupleBytes);
+ } else {
+ tupleWriter.writeTuple(tuple, minTupleBytes, 0);
+ }
+ ((ITreeIndexTupleReference) minTuple).resetByTupleOffset(minTupleBuf, 0);
+ }
+ }
+ if (maxTuple == null) {
+ int numBytes = tupleWriter.bytesRequired(tuple);
+ maxTupleBytes = new byte[numBytes];
+ tupleWriter.writeTuple(tuple, maxTupleBytes, 0);
+ maxTupleBuf = ByteBuffer.wrap(maxTupleBytes);
+ maxTuple = tupleWriter.createTupleReference();
+ ((ITreeIndexTupleReference) maxTuple).resetByTupleOffset(maxTupleBuf, 0);
+ } else {
+ int c = cmp.compare(tuple, maxTuple);
+ if (c > 0) {
+ int numBytes = tupleWriter.bytesRequired(tuple);
+ if (maxTupleBytes.length < numBytes) {
+ maxTupleBytes = new byte[numBytes];
+ tupleWriter.writeTuple(tuple, maxTupleBytes, 0);
+ maxTupleBuf = ByteBuffer.wrap(maxTupleBytes);
+ } else {
+ tupleWriter.writeTuple(tuple, maxTupleBytes, 0);
+ }
+ ((ITreeIndexTupleReference) maxTuple).resetByTupleOffset(maxTupleBuf, 0);
+ }
+ }
+ }
+
+ @Override
+ public ITupleReference getMinTuple() {
+ return minTuple;
+ }
+
+ @Override
+ public ITupleReference getMaxTuple() {
+ return maxTuple;
+ }
+
+ @Override
+ public boolean satisfy(ITupleReference minTuple, ITupleReference maxTuple, MultiComparator filterCmp) {
+ if (maxTuple != null && this.minTuple != null) {
+ int c = filterCmp.compare(maxTuple, this.minTuple);
+ if (c < 0) {
+ return false;
+ }
+ }
+ if (minTuple != null && this.maxTuple != null) {
+ int c = filterCmp.compare(minTuple, this.maxTuple);
+ if (c > 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMComponentFilterFactory.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMComponentFilterFactory.java
new file mode 100644
index 0000000..9968c7d
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMComponentFilterFactory.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
+
+public class LSMComponentFilterFactory implements ILSMComponentFilterFactory {
+
+ private final ITreeIndexTupleWriterFactory tupleWriterFactory;
+ private final IBinaryComparatorFactory[] filterCmpFactories;
+
+ public LSMComponentFilterFactory(ITreeIndexTupleWriterFactory tupleWriterFactory,
+ IBinaryComparatorFactory[] filterCmpFactories) {
+ this.tupleWriterFactory = tupleWriterFactory;
+ this.filterCmpFactories = filterCmpFactories;
+ }
+
+ @Override
+ public ILSMComponentFilter createLSMComponentFilter() {
+ return new LSMComponentFilter(tupleWriterFactory.createTupleWriter(), filterCmpFactories);
+ }
+}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
new file mode 100644
index 0000000..bd54769
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
@@ -0,0 +1,134 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.lsm.common.impls;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrame;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterManager;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+
+public class LSMComponentFilterManager implements ILSMComponentFilterManager {
+
+ private final IBufferCache bufferCache;
+ private final ILSMComponentFilterFrameFactory filterFrameFactory;
+
+ public LSMComponentFilterManager(IBufferCache bufferCache, ILSMComponentFilterFrameFactory filterFrameFactory) {
+ this.bufferCache = bufferCache;
+ this.filterFrameFactory = filterFrameFactory;
+ }
+
+ @Override
+ public void updateFilterInfo(ILSMComponentFilter filter, List<ITupleReference> filterTuples)
+ throws HyracksDataException {
+ MultiComparator filterCmp = MultiComparator.create(filter.getFilterCmpFactories());
+ for (ITupleReference tuple : filterTuples) {
+ filter.update(tuple, filterCmp);
+ }
+ }
+
+ @Override
+ public void writeFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex) throws HyracksDataException {
+ int fileId = treeIndex.getFileId();
+ ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory().createFrame();
+
+ // Read the filter page from the first metadata page of the tree.
+ // If it is has not been created yet, then create a new one.
+ int metadataPageId = treeIndex.getFreePageManager().getFirstMetadataPage();
+ ICachedPage metadataPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, metadataPageId), false);
+ metadataPage.acquireWriteLatch();
+ int componentFilterPageId;
+ try {
+ metadataFrame.setPage(metadataPage);
+ componentFilterPageId = metadataFrame.getLSMComponentFilterPageId();
+ if (componentFilterPageId == -1) {
+ componentFilterPageId = treeIndex.getFreePageManager().getFreePage(metadataFrame);
+ metadataFrame.setLSMComponentFilterPageId(componentFilterPageId);
+ }
+ } finally {
+ metadataPage.releaseWriteLatch(true);
+ bufferCache.unpin(metadataPage);
+ }
+
+ ICachedPage filterPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, componentFilterPageId), true);
+ filterPage.acquireWriteLatch();
+ try {
+ ILSMComponentFilterFrame filterFrame = filterFrameFactory.createFrame();
+ filterFrame.setPage(filterPage);
+ filterFrame.initBuffer();
+ if (filter.getMinTuple() != null) {
+ filterFrame.writeMinTuple(filter.getMinTuple());
+ }
+ if (filter.getMaxTuple() != null) {
+ filterFrame.writeMaxTuple(filter.getMaxTuple());
+ }
+
+ } finally {
+ filterPage.releaseWriteLatch(true);
+ bufferCache.unpin(filterPage);
+ }
+ }
+
+ @Override
+ public boolean readFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex) throws HyracksDataException {
+ int fileId = treeIndex.getFileId();
+ ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory().createFrame();
+
+ int metadataPageId = treeIndex.getFreePageManager().getFirstMetadataPage();
+ ICachedPage metadataPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, metadataPageId), false);
+ metadataPage.acquireReadLatch();
+ int componentFilterPageId;
+ try {
+ metadataFrame.setPage(metadataPage);
+ componentFilterPageId = metadataFrame.getLSMComponentFilterPageId();
+ if (componentFilterPageId == -1) {
+ return false;
+ }
+ } finally {
+ metadataPage.releaseReadLatch();
+ bufferCache.unpin(metadataPage);
+ }
+
+ ICachedPage filterPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, componentFilterPageId), true);
+ filterPage.acquireReadLatch();
+ try {
+ ILSMComponentFilterFrame filterFrame = filterFrameFactory.createFrame();
+ filterFrame.setPage(filterPage);
+
+ if (!filterFrame.isMinTupleSet() || !filterFrame.isMaxTupleSet()) {
+ return false;
+ }
+ List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ filterTuples.add(filterFrame.getMinTuple());
+ filterTuples.add(filterFrame.getMaxTuple());
+ updateFilterInfo(filter, filterTuples);
+
+ } finally {
+ filterPage.releaseReadLatch();
+ bufferCache.unpin(filterPage);
+ }
+ return true;
+ }
+}
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index a88b982..fd4442f 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -237,6 +237,7 @@
public void search(ILSMIndexOperationContext ctx, IIndexCursor cursor, ISearchPredicate pred)
throws HyracksDataException, IndexException {
LSMOperationType opType = LSMOperationType.SEARCH;
+ ctx.setSearchPredicate(pred);
getAndEnterComponents(ctx, opType, false);
try {
lsmIndex.search(ctx, cursor, pred);
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java
index ed2f8cf..2f04095 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java
@@ -17,6 +17,8 @@
import java.util.List;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.common.api.IIndex;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
@@ -35,20 +37,32 @@
public final class LSMInvertedIndexDataflowHelper extends AbstractLSMIndexDataflowHelper {
+ private final int[] invertedIndexFields;
+ private final int[] filterFieldsForNonBulkLoadOps;
+ private final int[] invertedIndexFieldsForNonBulkLoadOps;
+
public LSMInvertedIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
List<IVirtualBufferCache> virtualBufferCaches, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] invertedIndexFields,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
+ int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps) {
this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
- opTrackerFactory, ioScheduler, ioOpCallbackFactory);
+ opTrackerFactory, ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterTypeTraits,
+ filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps);
}
public LSMInvertedIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
List<IVirtualBufferCache> virtualBufferCaches, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory) {
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
+ int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps) {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
- ioScheduler, ioOpCallbackFactory);
+ ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields);
+ this.invertedIndexFields = invertedIndexFields;
+ this.filterFieldsForNonBulkLoadOps = filterFieldsForNonBulkLoadOps;
+ this.invertedIndexFieldsForNonBulkLoadOps = invertedIndexFieldsForNonBulkLoadOps;
}
@Override
@@ -63,7 +77,9 @@
invIndexOpDesc.getTokenComparatorFactories(), invIndexOpDesc.getTokenizerFactory(),
diskBufferCache, file.getFile().getPath(), bloomFilterFalsePositiveRate, mergePolicy,
opTrackerFactory.getOperationTracker(ctx), ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback());
+ ioOpCallbackFactory.createIOOperationCallback(), invertedIndexFields, filterTypeTraits,
+ filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
+ invertedIndexFieldsForNonBulkLoadOps);
return invIndex;
} catch (IndexException e) {
throw new HyracksDataException(e);
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
index c018f16..12c88ca 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java
@@ -18,6 +18,8 @@
import java.util.Map;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -30,13 +32,22 @@
public class LSMInvertedIndexDataflowHelperFactory extends AbstractLSMIndexDataflowHelperFactory {
private static final long serialVersionUID = 1L;
+ private final int[] invertedIndexFields;
+ private final int[] filterFieldsForNonBulkLoadOps;
+ private final int[] invertedIndexFieldsForNonBulkLoadOps;
public LSMInvertedIndexDataflowHelperFactory(IVirtualBufferCacheProvider virtualBufferCacheProvider,
ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
ILSMOperationTrackerProvider opTrackerProvider, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, double bloomFilterFalsePositiveRate) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, double bloomFilterFalsePositiveRate,
+ int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
+ int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps) {
super(virtualBufferCacheProvider, mergePolicyFactory, mergePolicyProperties, opTrackerProvider,
- ioSchedulerProvider, ioOpCallbackFactory, bloomFilterFalsePositiveRate);
+ ioSchedulerProvider, ioOpCallbackFactory, bloomFilterFalsePositiveRate, filterTypeTraits,
+ filterCmpFactories, filterFields);
+ this.invertedIndexFields = invertedIndexFields;
+ this.filterFieldsForNonBulkLoadOps = filterFieldsForNonBulkLoadOps;
+ this.invertedIndexFieldsForNonBulkLoadOps = invertedIndexFieldsForNonBulkLoadOps;
}
@Override
@@ -45,7 +56,8 @@
return new LSMInvertedIndexDataflowHelper(opDesc, ctx, partition,
virtualBufferCacheProvider.getVirtualBufferCaches(ctx), bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties), opTrackerFactory,
- ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory);
+ ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, invertedIndexFields, filterTypeTraits,
+ filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps);
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorDescriptor.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorDescriptor.java
index 0b33aac..47c68a2 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorDescriptor.java
@@ -40,6 +40,8 @@
private final int queryField;
private final IInvertedIndexSearchModifierFactory searchModifierFactory;
+ private final int[] minFilterFieldIndexes;
+ private final int[] maxFilterFieldIndexes;
public LSMInvertedIndexSearchOperatorDescriptor(IOperatorDescriptorRegistry spec, int queryField,
IStorageManagerInterface storageManager, IFileSplitProvider fileSplitProvider,
@@ -49,7 +51,9 @@
IIndexDataflowHelperFactory btreeDataflowHelperFactory, IBinaryTokenizerFactory queryTokenizerFactory,
IInvertedIndexSearchModifierFactory searchModifierFactory, RecordDescriptor recDesc, boolean retainInput,
boolean retainNull, INullWriterFactory nullWriterFactory,
- ISearchOperationCallbackFactory searchOpCallbackProvider) {
+ ISearchOperationCallbackFactory searchOpCallbackProvider, int[] minFilterFieldIndexes,
+ int[] maxFilterFieldIndexes) {
+
super(spec, 1, 1, recDesc, storageManager, fileSplitProvider, lifecycleManagerProvider, tokenTypeTraits,
tokenComparatorFactories, invListsTypeTraits, invListComparatorFactories, queryTokenizerFactory,
btreeDataflowHelperFactory, null, retainInput, retainNull, nullWriterFactory,
@@ -57,6 +61,8 @@
NoOpOperationCallbackFactory.INSTANCE);
this.queryField = queryField;
this.searchModifierFactory = searchModifierFactory;
+ this.minFilterFieldIndexes = minFilterFieldIndexes;
+ this.maxFilterFieldIndexes = maxFilterFieldIndexes;
}
@Override
@@ -64,6 +70,6 @@
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
IInvertedIndexSearchModifier searchModifier = searchModifierFactory.createSearchModifier();
return new LSMInvertedIndexSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, queryField,
- searchModifier);
+ searchModifier, minFilterFieldIndexes, maxFilterFieldIndexes);
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java
index 7a88be3..ddaeac0 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java
@@ -32,8 +32,8 @@
public LSMInvertedIndexSearchOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, IRecordDescriptorProvider recordDescProvider, int queryFieldIndex,
- IInvertedIndexSearchModifier searchModifier) {
- super(opDesc, ctx, partition, recordDescProvider);
+ IInvertedIndexSearchModifier searchModifier, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes) {
+ super(opDesc, ctx, partition, recordDescProvider, minFilterFieldIndexes, maxFilterFieldIndexes);
this.searchModifier = searchModifier;
this.queryFieldIndex = queryFieldIndex;
// If retainInput is true, the frameTuple is created in IndexSearchOperatorNodePushable.open().
@@ -47,7 +47,8 @@
@Override
protected ISearchPredicate createSearchPredicate() {
AbstractLSMInvertedIndexOperatorDescriptor invIndexOpDesc = (AbstractLSMInvertedIndexOperatorDescriptor) opDesc;
- return new InvertedIndexSearchPredicate(invIndexOpDesc.getTokenizerFactory().createTokenizer(), searchModifier);
+ return new InvertedIndexSearchPredicate(invIndexOpDesc.getTokenizerFactory().createTokenizer(), searchModifier,
+ minFilterKey, maxFilterKey);
}
@Override
@@ -56,6 +57,12 @@
InvertedIndexSearchPredicate invIndexSearchPred = (InvertedIndexSearchPredicate) searchPred;
invIndexSearchPred.setQueryTuple(frameTuple);
invIndexSearchPred.setQueryFieldIndex(queryFieldIndex);
+ if (minFilterKey != null) {
+ minFilterKey.reset(accessor, tupleIndex);
+ }
+ if (maxFilterKey != null) {
+ maxFilterKey.reset(accessor, tupleIndex);
+ }
}
@Override
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java
index 57a41ca..2ad2be2 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java
@@ -17,6 +17,8 @@
import java.util.List;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.common.api.IIndex;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
@@ -35,20 +37,32 @@
public final class PartitionedLSMInvertedIndexDataflowHelper extends AbstractLSMIndexDataflowHelper {
+ private final int[] invertedIndexFields;
+ private final int[] filterFieldsForNonBulkLoadOps;
+ private final int[] invertedIndexFieldsForNonBulkLoadOps;
+
public PartitionedLSMInvertedIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, List<IVirtualBufferCache> virtualBufferCache, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] invertedIndexFields,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
+ int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps) {
this(opDesc, ctx, partition, virtualBufferCache, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
- opTrackerFactory, ioScheduler, ioOpCallbackFactory);
+ opTrackerFactory, ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterTypeTraits,
+ filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps);
}
public PartitionedLSMInvertedIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, List<IVirtualBufferCache> virtualBufferCaches, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory) {
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
+ int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps) {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
- ioScheduler, ioOpCallbackFactory);
+ ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields);
+ this.invertedIndexFields = invertedIndexFields;
+ this.filterFieldsForNonBulkLoadOps = filterFieldsForNonBulkLoadOps;
+ this.invertedIndexFieldsForNonBulkLoadOps = invertedIndexFieldsForNonBulkLoadOps;
}
@Override
@@ -63,7 +77,9 @@
invIndexOpDesc.getTokenComparatorFactories(), invIndexOpDesc.getTokenizerFactory(),
diskBufferCache, file.getFile().getPath(), bloomFilterFalsePositiveRate, mergePolicy,
opTrackerFactory.getOperationTracker(ctx), ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback());
+ ioOpCallbackFactory.createIOOperationCallback(), invertedIndexFields, filterTypeTraits,
+ filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
+ invertedIndexFieldsForNonBulkLoadOps);
return invIndex;
} catch (IndexException e) {
throw new HyracksDataException(e);
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
index aef0863..b884b5d 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java
@@ -18,6 +18,8 @@
import java.util.Map;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -30,13 +32,22 @@
public class PartitionedLSMInvertedIndexDataflowHelperFactory extends AbstractLSMIndexDataflowHelperFactory {
private static final long serialVersionUID = 1L;
+ private final int[] invertedIndexFields;
+ private final int[] filterFieldsForNonBulkLoadOps;
+ private final int[] invertedIndexFieldsForNonBulkLoadOps;
public PartitionedLSMInvertedIndexDataflowHelperFactory(IVirtualBufferCacheProvider virtualBufferCacheProvider,
ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
ILSMOperationTrackerProvider opTrackerProvider, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, double bloomFilterFalsePositiveRate) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, double bloomFilterFalsePositiveRate,
+ int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
+ int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps) {
super(virtualBufferCacheProvider, mergePolicyFactory, mergePolicyProperties, opTrackerProvider,
- ioSchedulerProvider, ioOpCallbackFactory, bloomFilterFalsePositiveRate);
+ ioSchedulerProvider, ioOpCallbackFactory, bloomFilterFalsePositiveRate, filterTypeTraits,
+ filterCmpFactories, filterFields);
+ this.invertedIndexFields = invertedIndexFields;
+ this.filterFieldsForNonBulkLoadOps = filterFieldsForNonBulkLoadOps;
+ this.invertedIndexFieldsForNonBulkLoadOps = invertedIndexFieldsForNonBulkLoadOps;
}
@Override
@@ -45,6 +56,7 @@
return new PartitionedLSMInvertedIndexDataflowHelper(opDesc, ctx, partition,
virtualBufferCacheProvider.getVirtualBufferCaches(ctx), bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties), opTrackerFactory,
- ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory);
+ ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, invertedIndexFields, filterTypeTraits,
+ filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps);
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 44df9af..89e507b 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -43,6 +43,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IVirtualFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.exceptions.TreeIndexDuplicateKeyException;
+import edu.uci.ics.hyracks.storage.am.common.impls.AbstractSearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
@@ -50,6 +51,8 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -65,6 +68,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
@@ -91,26 +95,36 @@
protected final IBinaryComparatorFactory[] invListCmpFactories;
protected final ITypeTraits[] tokenTypeTraits;
protected final IBinaryComparatorFactory[] tokenCmpFactories;
+ private final int[] invertedIndexFields;
+ private final int[] filterFieldsForNonBulkLoadOps;
+ private final int[] invertedIndexFieldsForNonBulkLoadOps;
public LSMInvertedIndex(List<IVirtualBufferCache> virtualBufferCaches,
OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory,
- BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
- ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
+ BloomFilterFactory bloomFilterFactory, ILSMComponentFilterFactory filterFactory,
+ ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
+ double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager,
+ IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback) throws IndexException {
+ ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, int[] filterFields,
+ int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps) throws IndexException {
super(virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, diskFileMapProvider,
- bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback);
+ bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
+ filterManager, filterFields);
this.tokenizerFactory = tokenizerFactory;
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
this.tokenTypeTraits = tokenTypeTraits;
this.tokenCmpFactories = tokenCmpFactories;
+ this.invertedIndexFields = invertedIndexFields;
+ this.filterFieldsForNonBulkLoadOps = filterFieldsForNonBulkLoadOps;
+ this.invertedIndexFieldsForNonBulkLoadOps = invertedIndexFieldsForNonBulkLoadOps;
componentFactory = new LSMInvertedIndexDiskComponentFactory(diskInvIndexFactory, deletedKeysBTreeFactory,
- bloomFilterFactory);
+ bloomFilterFactory, filterFactory);
int i = 0;
for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
@@ -121,7 +135,8 @@
invListTypeTraits, invListCmpFactories, BTreeLeafFrameType.REGULAR_NSM, new FileReference(new File(
fileManager.getBaseDir() + "_virtual_del_" + i)));
LSMInvertedIndexMemoryComponent mutableComponent = new LSMInvertedIndexMemoryComponent(memInvIndex,
- deleteKeysBTree, virtualBufferCache, i == 0 ? true : false);
+ deleteKeysBTree, virtualBufferCache, i == 0 ? true : false, filterFactory == null ? null
+ : filterFactory.createLSMComponentFilter());
memoryComponents.add(mutableComponent);
++i;
}
@@ -209,8 +224,7 @@
isActivated = false;
if (flushOnExit) {
- BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(
- ioOpCallback);
+ BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
accessor.scheduleFlush(cb);
try {
@@ -288,7 +302,20 @@
}
// The current mutable component is always added
operationalComponents.add(0, memoryComponents.get(cmc));
- operationalComponents.addAll(immutableComponents);
+
+ if (filterManager != null) {
+ for (ILSMComponent c : immutableComponents) {
+ if (c.getLSMComponentFilter().satisfy(
+ ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMinFilterTuple(),
+ ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMaxFilterTuple(),
+ ((LSMInvertedIndexOpContext) ctx).filterCmp)) {
+ operationalComponents.add(c);
+ }
+ }
+ } else {
+ operationalComponents.addAll(immutableComponents);
+ }
+
break;
case MERGE:
operationalComponents.addAll(ctx.getComponentsToBeMerged());
@@ -324,19 +351,28 @@
// in the perspective of ASTERIX). The semantics for the operation callbacks
// are violated here (and they are somewhat unclear in the first place as to
// what they should be for an inverted index).
- ctx.modificationCallback.before(tuple);
- ctx.modificationCallback.found(null, tuple);
+
+ ITupleReference indexTuple;
+ if (ctx.indexTuple != null) {
+ ctx.indexTuple.reset(tuple);
+ indexTuple = ctx.indexTuple;
+ } else {
+ indexTuple = tuple;
+ }
+
+ ctx.modificationCallback.before(indexTuple);
+ ctx.modificationCallback.found(null, indexTuple);
switch (ctx.getOperation()) {
case INSERT: {
// Insert into the in-memory inverted index.
- ctx.currentMutableInvIndexAccessors.insert(tuple);
+ ctx.currentMutableInvIndexAccessors.insert(indexTuple);
break;
}
case DELETE: {
// First remove all entries in the in-memory inverted index (if any).
- ctx.currentMutableInvIndexAccessors.delete(tuple);
+ ctx.currentMutableInvIndexAccessors.delete(indexTuple);
// Insert key into the deleted-keys BTree.
- ctx.keysOnlyTuple.reset(tuple);
+ ctx.keysOnlyTuple.reset(indexTuple);
try {
ctx.currentDeletedKeysBTreeAccessors.insert(ctx.keysOnlyTuple);
} catch (TreeIndexDuplicateKeyException e) {
@@ -348,6 +384,11 @@
throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
}
}
+ if (ctx.filterTuple != null) {
+ ctx.filterTuple.reset(tuple);
+ memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter()
+ .update(ctx.filterTuple, ctx.filterCmp);
+ }
}
@Override
@@ -520,6 +561,15 @@
deletedKeysBTreeBulkLoader.end();
}
+ if (component.getLSMComponentFilter() != null) {
+ List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
+ filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
+ filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
+ filterManager.writeFilterInfo(component.getLSMComponentFilter(),
+ ((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
+ }
+
return component;
}
@@ -577,10 +627,11 @@
BTree btree = component.getDeletedKeysBTree();
IIndexBulkLoader btreeBulkLoader = btree.createBulkLoader(1.0f, true, 0L, false);
-
+
long numElements = 0L;
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
- numElements += ((LSMInvertedIndexDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter().getNumElements();
+ numElements += ((LSMInvertedIndexDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter()
+ .getNumElements();
}
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
@@ -613,6 +664,18 @@
cursor.close();
}
invIndexBulkLoader.end();
+
+ if (component.getLSMComponentFilter() != null) {
+ List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
+ filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
+ filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
+ }
+ filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
+ filterManager.writeFilterInfo(component.getLSMComponentFilter(),
+ ((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
+ }
+
return component;
}
@@ -637,6 +700,9 @@
private final IIndexBulkLoader invIndexBulkLoader;
private boolean cleanedUpArtifacts = false;
private boolean isEmptyComponent = true;
+ public final PermutingTupleReference indexTuple;
+ public final PermutingTupleReference filterTuple;
+ public final MultiComparator filterCmp;
public LSMInvertedIndexBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
boolean checkIfEmptyIndex) throws IndexException, HyracksDataException {
@@ -652,12 +718,36 @@
}
invIndexBulkLoader = ((LSMInvertedIndexDiskComponent) component).getInvIndex().createBulkLoader(fillFactor,
verifyInput, numElementsHint, false);
+
+ if (filterFields != null) {
+ indexTuple = new PermutingTupleReference(invertedIndexFields);
+ filterCmp = MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories());
+ filterTuple = new PermutingTupleReference(filterFields);
+ } else {
+ indexTuple = null;
+ filterCmp = null;
+ filterTuple = null;
+ }
}
@Override
public void add(ITupleReference tuple) throws IndexException, HyracksDataException {
try {
- invIndexBulkLoader.add(tuple);
+ ITupleReference t;
+ if (indexTuple != null) {
+ indexTuple.reset(tuple);
+ t = indexTuple;
+ } else {
+ t = tuple;
+ }
+
+ invIndexBulkLoader.add(t);
+
+ if (filterTuple != null) {
+ filterTuple.reset(tuple);
+ component.getLSMComponentFilter().update(filterTuple, filterCmp);
+ }
+
} catch (IndexException | HyracksDataException | RuntimeException e) {
cleanupArtifacts();
throw e;
@@ -683,6 +773,13 @@
public void end() throws IndexException, HyracksDataException {
if (!cleanedUpArtifacts) {
invIndexBulkLoader.end();
+
+ if (component.getLSMComponentFilter() != null) {
+ filterManager.writeFilterInfo(component.getLSMComponentFilter(),
+ (((OnDiskInvertedIndex) ((LSMInvertedIndexDiskComponent) component).getInvIndex())
+ .getBTree()));
+ }
+
if (isEmptyComponent) {
cleanupArtifacts();
} else {
@@ -714,6 +811,10 @@
component.getInvIndex().activate();
component.getDeletedKeysBTree().activate();
component.getBloomFilter().activate();
+ if (component.getLSMComponentFilter() != null) {
+ filterManager.readFilterInfo(component.getLSMComponentFilter(),
+ ((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
+ }
return component;
}
@@ -725,7 +826,8 @@
private LSMInvertedIndexOpContext createOpContext(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) throws HyracksDataException {
- return new LSMInvertedIndexOpContext(memoryComponents, modificationCallback, searchCallback);
+ return new LSMInvertedIndexOpContext(memoryComponents, modificationCallback, searchCallback,
+ invertedIndexFieldsForNonBulkLoadOps, filterFieldsForNonBulkLoadOps);
}
@Override
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java
index 446e807..6efefd6 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java
@@ -17,6 +17,7 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractDiskLSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex;
@@ -27,7 +28,9 @@
private final BTree deletedKeysBTree;
private final BloomFilter bloomFilter;
- public LSMInvertedIndexDiskComponent(IInvertedIndex invIndex, BTree deletedKeysBTree, BloomFilter bloomFilter) {
+ public LSMInvertedIndexDiskComponent(IInvertedIndex invIndex, BTree deletedKeysBTree, BloomFilter bloomFilter,
+ ILSMComponentFilter filter) {
+ super(filter);
this.invIndex = invIndex;
this.deletedKeysBTree = deletedKeysBTree;
this.bloomFilter = bloomFilter;
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java
index f83f529..8b6ce25 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponentFactory.java
@@ -21,6 +21,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory;
@@ -30,12 +31,15 @@
private final OnDiskInvertedIndexFactory diskInvIndexFactory;
private final TreeIndexFactory<BTree> btreeFactory;
private final BloomFilterFactory bloomFilterFactory;
+ private final ILSMComponentFilterFactory filterFactory;
public LSMInvertedIndexDiskComponentFactory(OnDiskInvertedIndexFactory diskInvIndexFactory,
- TreeIndexFactory<BTree> btreeFactory, BloomFilterFactory bloomFilterFactory) {
+ TreeIndexFactory<BTree> btreeFactory, BloomFilterFactory bloomFilterFactory,
+ ILSMComponentFilterFactory filterFactory) {
this.diskInvIndexFactory = diskInvIndexFactory;
this.btreeFactory = btreeFactory;
this.bloomFilterFactory = bloomFilterFactory;
+ this.filterFactory = filterFactory;
}
@Override
@@ -43,7 +47,8 @@
HyracksDataException {
return new LSMInvertedIndexDiskComponent(diskInvIndexFactory.createIndexInstance(cfr
.getInsertIndexFileReference()), btreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()),
- bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()));
+ bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()),
+ filterFactory == null ? null : filterFactory.createLSMComponentFilter());
}
@Override
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java
index eb1f915..b37b3b7 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMemoryComponent.java
@@ -17,6 +17,7 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractMemoryLSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
@@ -27,8 +28,8 @@
private final BTree deletedKeysBTree;
public LSMInvertedIndexMemoryComponent(IInvertedIndex invIndex, BTree deletedKeysBTree, IVirtualBufferCache vbc,
- boolean isActive) {
- super(vbc, isActive);
+ boolean isActive, ILSMComponentFilter filter) {
+ super(vbc, isActive, filter);
this.invIndex = invIndex;
this.deletedKeysBTree = deletedKeysBTree;
}
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
index 671e3f8..7a8305d 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
@@ -22,8 +22,10 @@
import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.tuples.PermutingTupleReference;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
@@ -36,7 +38,7 @@
private IndexOperation op;
private final List<ILSMComponent> componentHolder;
private final List<ILSMComponent> componentsToBeMerged;
-
+
public final IModificationOperationCallback modificationCallback;
public final ISearchOperationCallback searchCallback;
@@ -51,9 +53,15 @@
public IInvertedIndexAccessor currentMutableInvIndexAccessors;
public IIndexAccessor currentDeletedKeysBTreeAccessors;
+ public final PermutingTupleReference indexTuple;
+ public final MultiComparator filterCmp;
+ public final PermutingTupleReference filterTuple;
+
+ public ISearchPredicate searchPredicate;
+
public LSMInvertedIndexOpContext(List<ILSMComponent> mutableComponents,
- IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback)
- throws HyracksDataException {
+ IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback,
+ int[] invertedIndexFields, int[] filterFields) throws HyracksDataException {
this.componentHolder = new LinkedList<ILSMComponent>();
this.componentsToBeMerged = new LinkedList<ILSMComponent>();
this.modificationCallback = modificationCallback;
@@ -81,6 +89,16 @@
keyFieldPermutation[i] = NUM_DOCUMENT_FIELDS + i;
}
keysOnlyTuple = new PermutingTupleReference(keyFieldPermutation);
+
+ if (filterFields != null) {
+ indexTuple = new PermutingTupleReference(invertedIndexFields);
+ filterCmp = MultiComparator.create(c.getLSMComponentFilter().getFilterCmpFactories());
+ filterTuple = new PermutingTupleReference(filterFields);
+ } else {
+ indexTuple = null;
+ filterCmp = null;
+ filterTuple = null;
+ }
}
@Override
@@ -121,9 +139,19 @@
currentMutableInvIndexAccessors = mutableInvIndexAccessors[currentMutableComponentId];
currentDeletedKeysBTreeAccessors = deletedKeysBTreeAccessors[currentMutableComponentId];
}
-
+
@Override
public List<ILSMComponent> getComponentsToBeMerged() {
return componentsToBeMerged;
}
+
+ @Override
+ public void setSearchPredicate(ISearchPredicate searchPredicate) {
+ this.searchPredicate = searchPredicate;
+ }
+
+ @Override
+ public ISearchPredicate getSearchPredicate() {
+ return searchPredicate;
+ }
}
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
index ddb6060..eb78d10 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
@@ -24,6 +24,8 @@
import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import edu.uci.ics.hyracks.storage.am.common.api.IVirtualFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
@@ -31,6 +33,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInvertedIndex;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
@@ -41,15 +44,20 @@
public PartitionedLSMInvertedIndex(List<IVirtualBufferCache> virtualBufferCaches,
OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory,
- BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
- ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
+ BloomFilterFactory bloomFilterFactory, ILSMComponentFilterFactory filterFactory,
+ ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
+ double bloomFilterFalsePositiveRate, ILSMIndexFileManager fileManager,
+ IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback) throws IndexException {
- super(virtualBufferCaches, diskInvIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory,
- bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories,
- tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback);
+ ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, int[] filterFields,
+ int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps) throws IndexException {
+ super(virtualBufferCaches, diskInvIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterFactory,
+ filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider,
+ invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
+ mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields,
+ filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps);
}
@Override
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/InvertedIndexSearchPredicate.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/InvertedIndexSearchPredicate.java
index 2e83648..fa8939a 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/InvertedIndexSearchPredicate.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/InvertedIndexSearchPredicate.java
@@ -16,48 +16,55 @@
package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.search;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.impls.AbstractSearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifier;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
-public class InvertedIndexSearchPredicate implements ISearchPredicate {
+public class InvertedIndexSearchPredicate extends AbstractSearchPredicate {
private static final long serialVersionUID = 1L;
private ITupleReference queryTuple;
private int queryFieldIndex;
private final IBinaryTokenizer queryTokenizer;
- private final IInvertedIndexSearchModifier searchModifier;
-
+ private final IInvertedIndexSearchModifier searchModifier;
+
public InvertedIndexSearchPredicate(IBinaryTokenizer queryTokenizer, IInvertedIndexSearchModifier searchModifier) {
this.queryTokenizer = queryTokenizer;
this.searchModifier = searchModifier;
}
-
+
+ public InvertedIndexSearchPredicate(IBinaryTokenizer queryTokenizer, IInvertedIndexSearchModifier searchModifier,
+ ITupleReference minFilterTuple, ITupleReference maxFilterTuple) {
+ super(minFilterTuple, maxFilterTuple);
+ this.queryTokenizer = queryTokenizer;
+ this.searchModifier = searchModifier;
+ }
+
public void setQueryTuple(ITupleReference queryTuple) {
this.queryTuple = queryTuple;
}
-
+
public ITupleReference getQueryTuple() {
return queryTuple;
}
-
+
public void setQueryFieldIndex(int queryFieldIndex) {
this.queryFieldIndex = queryFieldIndex;
}
-
+
public int getQueryFieldIndex() {
return queryFieldIndex;
}
-
+
public IInvertedIndexSearchModifier getSearchModifier() {
return searchModifier;
}
-
+
public IBinaryTokenizer getQueryTokenizer() {
return queryTokenizer;
}
-
+
@Override
public MultiComparator getLowKeyComparator() {
// TODO: This doesn't make sense for an inverted index. Change ISearchPredicate interface.
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
index 372221e..9afe1e2 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
@@ -38,7 +38,10 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.common.frames.LSMComponentFilterFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFilterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilderFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndex;
@@ -122,7 +125,9 @@
IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory,
IBufferCache diskBufferCache, String onDiskDir, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback) throws IndexException {
+ ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps,
+ int[] invertedIndexFieldsForNonBulkLoadOps) throws IndexException {
BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(diskFileMapProvider, invListTypeTraits,
invListCmpFactories, diskBufferCache);
@@ -144,10 +149,21 @@
diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, fileManager);
+ LSMComponentFilterFactory filterFactory = null;
+ LSMComponentFilterFrameFactory filterFrameFactory = null;
+ LSMComponentFilterManager filterManager = null;
+ if (filterCmpFactories != null) {
+ TypeAwareTupleWriterFactory filterTupleWriterFactory = new TypeAwareTupleWriterFactory(filterTypeTraits);
+ filterFactory = new LSMComponentFilterFactory(filterTupleWriterFactory, filterCmpFactories);
+ filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory,
+ diskBufferCache.getPageSize());
+ filterManager = new LSMComponentFilterManager(diskBufferCache, filterFrameFactory);
+ }
LSMInvertedIndex invIndex = new LSMInvertedIndex(virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory,
- bloomFilterFactory, bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits,
- invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker,
- ioScheduler, ioOpCallback);
+ bloomFilterFactory, filterFactory, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate,
+ fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits,
+ tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback,
+ invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps);
return invIndex;
}
@@ -157,7 +173,9 @@
ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String onDiskDir,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback) throws IndexException {
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
+ int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps) throws IndexException {
BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(diskFileMapProvider, invListTypeTraits,
invListCmpFactories, diskBufferCache);
@@ -179,10 +197,22 @@
diskBufferCache, diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories,
tokenTypeTraits, tokenCmpFactories, fileManager);
+ LSMComponentFilterFactory filterFactory = null;
+ LSMComponentFilterFrameFactory filterFrameFactory = null;
+ LSMComponentFilterManager filterManager = null;
+ if (filterCmpFactories != null) {
+ TypeAwareTupleWriterFactory filterTupleWriterFactory = new TypeAwareTupleWriterFactory(filterTypeTraits);
+ filterFactory = new LSMComponentFilterFactory(filterTupleWriterFactory, filterCmpFactories);
+ filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory,
+ diskBufferCache.getPageSize());
+ filterManager = new LSMComponentFilterManager(diskBufferCache, filterFrameFactory);
+ }
PartitionedLSMInvertedIndex invIndex = new PartitionedLSMInvertedIndex(virtualBufferCaches, invIndexFactory,
- deletedKeysBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, fileManager,
- diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories,
- tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback);
+ deletedKeysBTreeFactory, bloomFilterFactory, filterFactory, filterFrameFactory, filterManager,
+ bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories,
+ tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler,
+ ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps,
+ invertedIndexFieldsForNonBulkLoadOps);
return invIndex;
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
index 636f0cb..41f8e5c 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
@@ -44,16 +44,19 @@
protected final IPrimitiveValueProviderFactory[] valueProviderFactories;
protected final RTreePolicyType rtreePolicyType;
protected final ILinearizeComparatorFactory linearizeCmpFactory;
+ protected final int[] rtreeFields;
public AbstractLSMRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
List<IVirtualBufferCache> virtualBufferCaches, IBinaryComparatorFactory[] btreeComparatorFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- ILinearizeComparatorFactory linearizeCmpFactory) {
+ ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) {
this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE,
btreeComparatorFactories, valueProviderFactories, rtreePolicyType, mergePolicy, opTrackerFactory,
- ioScheduler, ioOpCallbackFactory, linearizeCmpFactory);
+ ioScheduler, ioOpCallbackFactory, linearizeCmpFactory, rtreeFields, filterTypeTraits,
+ filterCmpFactories, filterFields);
}
public AbstractLSMRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
@@ -62,13 +65,15 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- ILinearizeComparatorFactory linearizeCmpFactory) {
+ ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
- ioScheduler, ioOpCallbackFactory);
+ ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields);
this.btreeComparatorFactories = btreeComparatorFactories;
this.valueProviderFactories = valueProviderFactories;
this.rtreePolicyType = rtreePolicyType;
this.linearizeCmpFactory = linearizeCmpFactory;
+ this.rtreeFields = rtreeFields;
}
@Override
@@ -77,7 +82,8 @@
return createLSMTree(virtualBufferCaches, file, opDesc.getStorageManager().getBufferCache(ctx), opDesc
.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
treeOpDesc.getTreeIndexComparatorFactories(), btreeComparatorFactories,
- opTrackerFactory.getOperationTracker(ctx), valueProviderFactories, rtreePolicyType, linearizeCmpFactory);
+ opTrackerFactory.getOperationTracker(ctx), valueProviderFactories, rtreePolicyType,
+ linearizeCmpFactory, rtreeFields, filterTypeTraits, filterCmpFactories, filterFields);
}
@@ -85,6 +91,7 @@
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
ILSMOperationTracker opTracker, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory)
+ RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields)
throws HyracksDataException;
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
index 7b823b3..391b066 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
@@ -49,7 +49,8 @@
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
ILinearizeComparatorFactory linearizeCmpFactory, int[] btreeFields, int version) {
super(opDesc, ctx, partition, null, btreeComparatorFactories, valueProviderFactories, rtreePolicyType,
- mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, linearizeCmpFactory, btreeFields);
+ mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, linearizeCmpFactory, null,
+ btreeFields, null, null, null);
this.version = version;
}
@@ -61,7 +62,7 @@
ILinearizeComparatorFactory linearizeCmpFactory, int[] btreeFields, int version) {
super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, btreeComparatorFactories,
valueProviderFactories, rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler,
- ioOpCallbackFactory, linearizeCmpFactory, btreeFields);
+ ioOpCallbackFactory, linearizeCmpFactory, null, btreeFields, null, null, null);
this.version = version;
}
@@ -90,7 +91,8 @@
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
ILSMOperationTracker opTracker, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory)
+ RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields)
throws HyracksDataException {
try {
return LSMRTreeUtils.createExternalRTree(file, diskBufferCache, diskFileMapProvider, typeTraits,
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelperFactory.java
index 81b2582..ca4035a 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelperFactory.java
@@ -41,7 +41,7 @@
double bloomFilterFalsePositiveRate, int[] btreeFields, int version) {
super(valueProviderFactories, rtreePolicyType, btreeComparatorFactories, null, mergePolicyFactory,
mergePolicyProperties, opTrackerFactory, ioSchedulerProvider, ioOpCallbackFactory, linearizeCmpFactory,
- bloomFilterFalsePositiveRate, btreeFields);
+ bloomFilterFalsePositiveRate, null, btreeFields, null, null, null);
this.version = version;
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
index f316503..d9cfe5d 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
@@ -46,9 +46,11 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- ILinearizeComparatorFactory linearizeCmpFactory, int[] btreeFields) {
+ ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, int[] btreeFields,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) {
super(opDesc, ctx, partition, virtualBufferCaches, btreeComparatorFactories, valueProviderFactories,
- rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, linearizeCmpFactory);
+ rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, linearizeCmpFactory,
+ rtreeFields, filterTypeTraits, filterCmpFactories, filterFields);
this.btreeFields = btreeFields;
}
@@ -58,10 +60,12 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- ILinearizeComparatorFactory linearizeCmpFactory, int[] btreeFields) {
+ ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, int[] btreeFields,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, btreeComparatorFactories,
valueProviderFactories, rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler,
- ioOpCallbackFactory, linearizeCmpFactory);
+ ioOpCallbackFactory, linearizeCmpFactory, rtreeFields, filterTypeTraits, filterCmpFactories,
+ filterFields);
this.btreeFields = btreeFields;
}
@@ -70,13 +74,15 @@
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
ILSMOperationTracker opTracker, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory)
+ RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields)
throws HyracksDataException {
try {
return LSMRTreeUtils.createLSMTree(virtualBufferCaches, file, diskBufferCache, diskFileMapProvider,
typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory, btreeFields);
+ ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory, rtreeFields, btreeFields,
+ filterTypeTraits, filterCmpFactories, filterFields);
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
index 7548398..4fb3718 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
@@ -20,6 +20,7 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
@@ -39,6 +40,7 @@
protected final IPrimitiveValueProviderFactory[] valueProviderFactories;
protected final RTreePolicyType rtreePolicyType;
protected final ILinearizeComparatorFactory linearizeCmpFactory;
+ protected final int[] rtreeFields;
protected final int[] btreeFields;
public LSMRTreeDataflowHelperFactory(IPrimitiveValueProviderFactory[] valueProviderFactories,
@@ -46,13 +48,17 @@
IVirtualBufferCacheProvider virtualBufferCacheProvider, ILSMMergePolicyFactory mergePolicyFactory,
Map<String, String> mergePolicyProperties, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- ILinearizeComparatorFactory linearizeCmpFactory, double bloomFilterFalsePositiveRate, int[] btreeFields) {
+ ILinearizeComparatorFactory linearizeCmpFactory, double bloomFilterFalsePositiveRate, int[] rtreeFields,
+ int[] btreeFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
+ int[] filterFields) {
super(virtualBufferCacheProvider, mergePolicyFactory, mergePolicyProperties, opTrackerFactory,
- ioSchedulerProvider, ioOpCallbackFactory, bloomFilterFalsePositiveRate);
+ ioSchedulerProvider, ioOpCallbackFactory, bloomFilterFalsePositiveRate, filterTypeTraits,
+ filterCmpFactories, filterFields);
this.btreeComparatorFactories = btreeComparatorFactories;
this.valueProviderFactories = valueProviderFactories;
this.rtreePolicyType = rtreePolicyType;
this.linearizeCmpFactory = linearizeCmpFactory;
+ this.rtreeFields = rtreeFields;
this.btreeFields = btreeFields;
}
@@ -63,6 +69,7 @@
virtualBufferCacheProvider.getVirtualBufferCaches(ctx), bloomFilterFalsePositiveRate,
btreeComparatorFactories, valueProviderFactories, rtreePolicyType,
mergePolicyFactory.createMergePolicy(mergePolicyProperties), opTrackerFactory,
- ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, linearizeCmpFactory, btreeFields);
+ ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, linearizeCmpFactory, rtreeFields,
+ btreeFields, filterTypeTraits, filterCmpFactories, filterFields);
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
index f430456..2029075 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelper.java
@@ -45,9 +45,11 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- ILinearizeComparatorFactory linearizeCmpFactory) {
+ ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) {
super(opDesc, ctx, partition, virtualBufferCaches, btreeComparatorFactories, valueProviderFactories,
- rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, linearizeCmpFactory);
+ rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, linearizeCmpFactory, rtreeFields, filterTypeTraits,
+ filterCmpFactories, filterFields);
}
@Override
@@ -55,13 +57,15 @@
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
ILSMOperationTracker opTracker, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory)
+ RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields)
throws HyracksDataException {
try {
return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(virtualBufferCaches, file, diskBufferCache,
diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories,
rtreePolicyType, mergePolicy, opTracker, ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory);
+ ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory, rtreeFields,
+ filterTypeTraits, filterCmpFactories, filterFields);
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
index 2e1cfaa..a7dc293 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterTuplesDataflowHelperFactory.java
@@ -20,6 +20,7 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
@@ -39,19 +40,22 @@
private final IPrimitiveValueProviderFactory[] valueProviderFactories;
private final RTreePolicyType rtreePolicyType;
private final ILinearizeComparatorFactory linearizeCmpFactory;
+ private final int[] rtreeFields;
public LSMRTreeWithAntiMatterTuplesDataflowHelperFactory(IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, IBinaryComparatorFactory[] btreeComparatorFactories,
IVirtualBufferCacheProvider virtualBufferCacheProvider, ILSMMergePolicyFactory mergePolicyFactory,
Map<String, String> mergePolicyProperties, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- ILinearizeComparatorFactory linearizeCmpFactory) {
- super(virtualBufferCacheProvider, mergePolicyFactory, mergePolicyProperties, opTrackerFactory, ioSchedulerProvider,
- ioOpCallbackFactory, 1.0);
+ ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) {
+ super(virtualBufferCacheProvider, mergePolicyFactory, mergePolicyProperties, opTrackerFactory,
+ ioSchedulerProvider, ioOpCallbackFactory, 1.0, filterTypeTraits, filterCmpFactories, filterFields);
this.btreeComparatorFactories = btreeComparatorFactories;
this.valueProviderFactories = valueProviderFactories;
this.rtreePolicyType = rtreePolicyType;
this.linearizeCmpFactory = linearizeCmpFactory;
+ this.rtreeFields = rtreeFields;
}
@Override
@@ -60,6 +64,7 @@
return new LSMRTreeWithAntiMatterTuplesDataflowHelper(opDesc, ctx, partition,
virtualBufferCacheProvider.getVirtualBufferCaches(ctx), btreeComparatorFactories,
valueProviderFactories, rtreePolicyType, mergePolicyFactory.createMergePolicy(mergePolicyProperties),
- opTrackerFactory, ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, linearizeCmpFactory);
+ opTrackerFactory, ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, linearizeCmpFactory,
+ rtreeFields, filterTypeTraits, filterCmpFactories, filterFields);
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index a69f1f8..09d0558 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -33,10 +33,13 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.exceptions.TreeIndexDuplicateKeyException;
+import edu.uci.ics.hyracks.storage.am.common.impls.AbstractSearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
@@ -49,6 +52,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
@@ -73,6 +77,7 @@
protected final ITreeIndexFrameFactory btreeInteriorFrameFactory;
protected final ITreeIndexFrameFactory rtreeLeafFrameFactory;
protected final ITreeIndexFrameFactory btreeLeafFrameFactory;
+ protected final int[] rtreeFields;
public AbstractLSMRTree(List<IVirtualBufferCache> virtualBufferCaches,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
@@ -82,9 +87,12 @@
IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback) {
+ ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFactory filterFactory,
+ ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
+ int[] rtreeFields, int[] filterFields) {
super(virtualBufferCaches, componentFactory.getBufferCache(), fileManager, diskFileMapProvider,
- bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback);
+ bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
+ filterManager, filterFields);
int i = 0;
for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
RTree memRTree = new RTree(virtualBufferCache,
@@ -98,7 +106,8 @@
btreeCmpFactories, btreeCmpFactories.length, new FileReference(new File(fileManager.getBaseDir()
+ "_virtual_b_" + i)));
LSMRTreeMemoryComponent mutableComponent = new LSMRTreeMemoryComponent(memRTree, memBTree,
- virtualBufferCache, i == 0 ? true : false);
+ virtualBufferCache, i == 0 ? true : false, filterFactory == null ? null
+ : filterFactory.createLSMComponentFilter());
memoryComponents.add(mutableComponent);
++i;
}
@@ -113,6 +122,7 @@
this.linearizer = linearizer;
this.comparatorFields = comparatorFields;
this.linearizerArray = linearizerArray;
+ this.rtreeFields = rtreeFields;
}
/*
@@ -138,6 +148,7 @@
this.linearizer = linearizer;
this.comparatorFields = comparatorFields;
this.linearizerArray = linearizerArray;
+ this.rtreeFields = null;
}
@Override
@@ -240,7 +251,20 @@
}
// The current mutable component is always added
operationalComponents.add(0, memoryComponents.get(cmc));
- operationalComponents.addAll(immutableComponents);
+
+ if (filterManager != null) {
+ for (ILSMComponent c : immutableComponents) {
+ if (c.getLSMComponentFilter().satisfy(
+ ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMinFilterTuple(),
+ ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMaxFilterTuple(),
+ ((LSMRTreeOpContext) ctx).filterCmp)) {
+ operationalComponents.add(c);
+ }
+ }
+ } else {
+ operationalComponents.addAll(immutableComponents);
+ }
+
break;
case MERGE:
operationalComponents.addAll(ctx.getComponentsToBeMerged());
@@ -298,6 +322,9 @@
component.getBTree().activate();
component.getBloomFilter().activate();
}
+ if (component.getLSMComponentFilter() != null) {
+ filterManager.readFilterInfo(component.getLSMComponentFilter(), component.getRTree());
+ }
return component;
}
@@ -350,28 +377,41 @@
throw new UnsupportedOperationException("Physical delete not supported in the LSM-RTree");
}
- ctx.modificationCallback.before(tuple);
- ctx.modificationCallback.found(null, tuple);
+ ITupleReference indexTuple;
+ if (ctx.indexTuple != null) {
+ ctx.indexTuple.reset(tuple);
+ indexTuple = ctx.indexTuple;
+ } else {
+ indexTuple = tuple;
+ }
+
+ ctx.modificationCallback.before(indexTuple);
+ ctx.modificationCallback.found(null, indexTuple);
if (ctx.getOperation() == IndexOperation.INSERT) {
- ctx.currentMutableRTreeAccessor.insert(tuple);
+ ctx.currentMutableRTreeAccessor.insert(indexTuple);
} else {
// First remove all entries in the in-memory rtree (if any).
- ctx.currentMutableRTreeAccessor.delete(tuple);
+ ctx.currentMutableRTreeAccessor.delete(indexTuple);
// Insert key into the deleted-keys BTree.
try {
- ctx.currentMutableBTreeAccessor.insert(tuple);
+ ctx.currentMutableBTreeAccessor.insert(indexTuple);
} catch (TreeIndexDuplicateKeyException e) {
// Do nothing, because one delete tuple is enough to indicate
// that all the corresponding insert tuples are deleted
}
}
+ if (ctx.filterTuple != null) {
+ ctx.filterTuple.reset(tuple);
+ memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter()
+ .update(ctx.filterTuple, ctx.filterCmp);
+ }
}
protected LSMRTreeOpContext createOpContext(IModificationOperationCallback modCallback) {
return new LSMRTreeOpContext(memoryComponents, (IRTreeLeafFrame) rtreeLeafFrameFactory.createFrame(),
(IRTreeInteriorFrame) rtreeInteriorFrameFactory.createFrame(), btreeLeafFrameFactory,
btreeInteriorFrameFactory, rtreeCmpFactories, btreeCmpFactories, modCallback,
- NoOpOperationCallback.INSTANCE);
+ NoOpOperationCallback.INSTANCE, rtreeFields, filterFields);
}
@Override
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
index 55d0cad..0ce83dc 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
@@ -20,84 +20,95 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
public class ExternalRTreeOpContext implements ILSMIndexOperationContext {
- private IndexOperation op;
- private MultiComparator bTreeCmp;
- private MultiComparator rTreeCmp;
- public final List<ILSMComponent> componentHolder;
- private final List<ILSMComponent> componentsToBeMerged;
- public final ISearchOperationCallback searchCallback;
- private final int targetIndexVersion;
+ private IndexOperation op;
+ private MultiComparator bTreeCmp;
+ private MultiComparator rTreeCmp;
+ public final List<ILSMComponent> componentHolder;
+ private final List<ILSMComponent> componentsToBeMerged;
+ public final ISearchOperationCallback searchCallback;
+ private final int targetIndexVersion;
+ public ISearchPredicate searchPredicate;
- public ExternalRTreeOpContext(
- IBinaryComparatorFactory[] rtreeCmpFactories,
- IBinaryComparatorFactory[] btreeCmpFactories,
- ISearchOperationCallback searchCallback, int targetIndexVersion) {
+ public ExternalRTreeOpContext(IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeCmpFactories, ISearchOperationCallback searchCallback,
+ int targetIndexVersion) {
- this.componentHolder = new LinkedList<ILSMComponent>();
- this.componentsToBeMerged = new LinkedList<ILSMComponent>();
- this.searchCallback = searchCallback;
- this.targetIndexVersion = targetIndexVersion;
- this.bTreeCmp = MultiComparator.create(btreeCmpFactories);
- this.rTreeCmp = MultiComparator.create(rtreeCmpFactories);
- }
+ this.componentHolder = new LinkedList<ILSMComponent>();
+ this.componentsToBeMerged = new LinkedList<ILSMComponent>();
+ this.searchCallback = searchCallback;
+ this.targetIndexVersion = targetIndexVersion;
+ this.bTreeCmp = MultiComparator.create(btreeCmpFactories);
+ this.rTreeCmp = MultiComparator.create(rtreeCmpFactories);
+ }
- public void setOperation(IndexOperation newOp) {
- reset();
- this.op = newOp;
- }
+ public void setOperation(IndexOperation newOp) {
+ reset();
+ this.op = newOp;
+ }
- @Override
- public void setCurrentMutableComponentId(int currentMutableComponentId) {
- // Do nothing. this should never be called for disk only indexes
- }
+ @Override
+ public void setCurrentMutableComponentId(int currentMutableComponentId) {
+ // Do nothing. this should never be called for disk only indexes
+ }
- @Override
- public void reset() {
- componentHolder.clear();
- componentsToBeMerged.clear();
- }
+ @Override
+ public void reset() {
+ componentHolder.clear();
+ componentsToBeMerged.clear();
+ }
- @Override
- public IndexOperation getOperation() {
- return op;
- }
+ @Override
+ public IndexOperation getOperation() {
+ return op;
+ }
- public MultiComparator getBTreeMultiComparator() {
- return bTreeCmp;
- }
+ public MultiComparator getBTreeMultiComparator() {
+ return bTreeCmp;
+ }
- public MultiComparator getRTreeMultiComparator() {
- return rTreeCmp;
- }
+ public MultiComparator getRTreeMultiComparator() {
+ return rTreeCmp;
+ }
- @Override
- public List<ILSMComponent> getComponentHolder() {
- return componentHolder;
- }
+ @Override
+ public List<ILSMComponent> getComponentHolder() {
+ return componentHolder;
+ }
- @Override
- public ISearchOperationCallback getSearchOperationCallback() {
- return searchCallback;
- }
+ @Override
+ public ISearchOperationCallback getSearchOperationCallback() {
+ return searchCallback;
+ }
- // This should never be needed for disk only indexes
- @Override
- public IModificationOperationCallback getModificationCallback() {
- return null;
- }
+ // This should never be needed for disk only indexes
+ @Override
+ public IModificationOperationCallback getModificationCallback() {
+ return null;
+ }
- @Override
- public List<ILSMComponent> getComponentsToBeMerged() {
- return componentsToBeMerged;
- }
+ @Override
+ public List<ILSMComponent> getComponentsToBeMerged() {
+ return componentsToBeMerged;
+ }
- public int getTargetIndexVersion() {
- return targetIndexVersion;
- }
+ public int getTargetIndexVersion() {
+ return targetIndexVersion;
+ }
+
+ @Override
+ public void setSearchPredicate(ISearchPredicate searchPredicate) {
+ this.searchPredicate = searchPredicate;
+ }
+
+ @Override
+ public ISearchPredicate getSearchPredicate() {
+ return searchPredicate;
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 37ce427..3e092aa 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -15,6 +15,7 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
+import java.util.ArrayList;
import java.util.List;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -45,7 +46,10 @@
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.tuples.DualTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.tuples.PermutingTupleReference;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
@@ -57,6 +61,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
@@ -72,17 +77,19 @@
ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory,
ITreeIndexFrameFactory btreeLeafFrameFactory, ILSMIndexFileManager fileNameManager,
TreeIndexFactory<RTree> diskRTreeFactory, TreeIndexFactory<BTree> diskBTreeFactory,
- BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
- IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories,
- IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
- int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMMergePolicy mergePolicy,
- ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
- int[] buddyBTreeFields) {
+ BloomFilterFactory bloomFilterFactory, ILSMComponentFilterFactory filterFactory,
+ ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
+ double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount,
+ IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
+ ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallback ioOpCallback, int[] rtreeFields, int[] buddyBTreeFields, int[] filterFields) {
super(virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory,
btreeLeafFrameFactory, fileNameManager, new LSMRTreeDiskComponentFactory(diskRTreeFactory,
- diskBTreeFactory, bloomFilterFactory), diskFileMapProvider, fieldCount, rtreeCmpFactories,
- btreeCmpFactories, linearizer, comparatorFields, linearizerArray, bloomFilterFalsePositiveRate,
- mergePolicy, opTracker, ioScheduler, ioOpCallback);
+ diskBTreeFactory, bloomFilterFactory, filterFactory), diskFileMapProvider, fieldCount,
+ rtreeCmpFactories, btreeCmpFactories, linearizer, comparatorFields, linearizerArray,
+ bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFactory,
+ filterFrameFactory, filterManager, rtreeFields, filterFields);
this.buddyBTreeFields = buddyBTreeFields;
}
@@ -100,7 +107,7 @@
ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields) {
super(rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory,
fileNameManager, new LSMRTreeDiskComponentFactory(diskRTreeFactory, diskBTreeFactory,
- bloomFilterFactory), diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories,
+ bloomFilterFactory, null), diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories,
linearizer, comparatorFields, linearizerArray, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
ioScheduler, ioOpCallback);
this.buddyBTreeFields = buddyBTreeFields;
@@ -305,6 +312,14 @@
bTreeBulkloader.end();
}
+ if (component.getLSMComponentFilter() != null) {
+ List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
+ filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
+ filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
+ filterManager.writeFilterInfo(component.getLSMComponentFilter(), component.getRTree());
+ }
+
return component;
}
@@ -384,6 +399,17 @@
cursor.close();
}
bulkLoader.end();
+
+ if (mergedComponent.getLSMComponentFilter() != null) {
+ List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
+ filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
+ filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
+ }
+ filterManager.updateFilterInfo(mergedComponent.getLSMComponentFilter(), filterTuples);
+ filterManager.writeFilterInfo(mergedComponent.getLSMComponentFilter(), mergedComponent.getRTree());
+ }
+
return mergedComponent;
}
@@ -394,10 +420,11 @@
}
public class LSMRTreeAccessor extends LSMTreeIndexAccessor {
- private DualTupleReference dualTuple = new DualTupleReference(buddyBTreeFields);
+ private final DualTupleReference dualTuple;
public LSMRTreeAccessor(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) {
super(lsmHarness, ctx);
+ dualTuple = new DualTupleReference(buddyBTreeFields);
}
@Override
@@ -407,15 +434,22 @@
@Override
public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
- dualTuple.reset(tuple);
ctx.setOperation(IndexOperation.DELETE);
+ dualTuple.reset(tuple);
lsmHarness.modify(ctx, false, dualTuple);
}
@Override
- public void forceDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
- dualTuple.reset(tuple);
+ public boolean tryDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
ctx.setOperation(IndexOperation.DELETE);
+ dualTuple.reset(tuple);
+ return lsmHarness.modify(ctx, true, dualTuple);
+ }
+
+ @Override
+ public void forceDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ ctx.setOperation(IndexOperation.DELETE);
+ dualTuple.reset(tuple);
lsmHarness.forceModify(ctx, dualTuple);
}
@@ -449,13 +483,21 @@
throw new UnsupportedOperationException("Physical delete not supported in the LSM-RTree");
}
- ctx.modificationCallback.before(tuple);
- ctx.modificationCallback.found(null, tuple);
+ ITupleReference indexTuple;
+ if (ctx.indexTuple != null) {
+ ctx.indexTuple.reset(tuple);
+ indexTuple = ctx.indexTuple;
+ } else {
+ indexTuple = tuple;
+ }
+
+ ctx.modificationCallback.before(indexTuple);
+ ctx.modificationCallback.found(null, indexTuple);
if (ctx.getOperation() == IndexOperation.INSERT) {
- ctx.currentMutableRTreeAccessor.insert(tuple);
+ ctx.currentMutableRTreeAccessor.insert(indexTuple);
} else {
// First remove all entries in the in-memory rtree (if any).
- ctx.currentMutableRTreeAccessor.delete(tuple);
+ ctx.currentMutableRTreeAccessor.delete(indexTuple);
try {
ctx.currentMutableBTreeAccessor.insert(((DualTupleReference) tuple).getPermutingTuple());
} catch (TreeIndexDuplicateKeyException e) {
@@ -463,6 +505,11 @@
// that all the corresponding insert tuples are deleted
}
}
+ if (ctx.filterTuple != null) {
+ ctx.filterTuple.reset(tuple);
+ memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter()
+ .update(ctx.filterTuple, ctx.filterCmp);
+ }
}
public class LSMRTreeBulkLoader implements IIndexBulkLoader {
@@ -470,6 +517,9 @@
private final IIndexBulkLoader bulkLoader;
private boolean cleanedUpArtifacts = false;
private boolean isEmptyComponent = true;
+ public final PermutingTupleReference indexTuple;
+ public final PermutingTupleReference filterTuple;
+ public final MultiComparator filterCmp;
public LSMRTreeBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex)
throws TreeIndexException, HyracksDataException {
@@ -485,12 +535,35 @@
}
bulkLoader = ((LSMRTreeDiskComponent) component).getRTree().createBulkLoader(fillFactor, verifyInput,
numElementsHint, false);
+
+ if (filterFields != null) {
+ indexTuple = new PermutingTupleReference(rtreeFields);
+ filterCmp = MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories());
+ filterTuple = new PermutingTupleReference(filterFields);
+ } else {
+ indexTuple = null;
+ filterCmp = null;
+ filterTuple = null;
+ }
}
@Override
public void add(ITupleReference tuple) throws HyracksDataException, IndexException {
try {
- bulkLoader.add(tuple);
+ ITupleReference t;
+ if (indexTuple != null) {
+ indexTuple.reset(tuple);
+ t = indexTuple;
+ } else {
+ t = tuple;
+ }
+
+ bulkLoader.add(t);
+
+ if (filterTuple != null) {
+ filterTuple.reset(tuple);
+ component.getLSMComponentFilter().update(filterTuple, filterCmp);
+ }
} catch (IndexException | HyracksDataException | RuntimeException e) {
cleanupArtifacts();
throw e;
@@ -504,6 +577,12 @@
public void end() throws HyracksDataException, IndexException {
if (!cleanedUpArtifacts) {
bulkLoader.end();
+
+ if (component.getLSMComponentFilter() != null) {
+ filterManager.writeFilterInfo(component.getLSMComponentFilter(),
+ ((LSMRTreeDiskComponent) component).getRTree());
+ }
+
if (isEmptyComponent) {
cleanupArtifacts();
} else {
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java
index c3c0a55..ecf8e59 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java
@@ -17,6 +17,7 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractDiskLSMComponent;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
@@ -25,7 +26,8 @@
private final BTree btree;
private final BloomFilter bloomFilter;
- public LSMRTreeDiskComponent(RTree rtree, BTree btree, BloomFilter bloomFilter) {
+ public LSMRTreeDiskComponent(RTree rtree, BTree btree, BloomFilter bloomFilter, ILSMComponentFilter filter) {
+ super(filter);
this.rtree = rtree;
this.btree = btree;
this.bloomFilter = bloomFilter;
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentFactory.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentFactory.java
index a130562..2ab4d47 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponentFactory.java
@@ -21,6 +21,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
@@ -30,12 +31,14 @@
private final TreeIndexFactory<RTree> rtreeFactory;
private final TreeIndexFactory<BTree> btreeFactory;
private final BloomFilterFactory bloomFilterFactory;
+ private final ILSMComponentFilterFactory filterFactory;
public LSMRTreeDiskComponentFactory(TreeIndexFactory<RTree> rtreeFactory, TreeIndexFactory<BTree> btreeFactory,
- BloomFilterFactory bloomFilterFactory) {
+ BloomFilterFactory bloomFilterFactory, ILSMComponentFilterFactory filterFactory) {
this.rtreeFactory = rtreeFactory;
this.btreeFactory = btreeFactory;
this.bloomFilterFactory = bloomFilterFactory;
+ this.filterFactory = filterFactory;
}
@Override
@@ -43,7 +46,8 @@
HyracksDataException {
return new LSMRTreeDiskComponent(rtreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
btreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()),
- bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()));
+ bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()),
+ filterFactory == null ? null : filterFactory.createLSMComponentFilter());
}
@Override
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMemoryComponent.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMemoryComponent.java
index 452643b..77c2739 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMemoryComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMemoryComponent.java
@@ -17,6 +17,7 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractMemoryLSMComponent;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
@@ -26,8 +27,9 @@
private final RTree rtree;
private final BTree btree;
- public LSMRTreeMemoryComponent(RTree rtree, BTree btree, IVirtualBufferCache vbc, boolean isActive) {
- super(vbc, isActive);
+ public LSMRTreeMemoryComponent(RTree rtree, BTree btree, IVirtualBufferCache vbc, boolean isActive,
+ ILSMComponentFilter filter) {
+ super(vbc, isActive, filter);
this.rtree = rtree;
this.btree = btree;
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
index 132e55b..b2c939a 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
@@ -23,10 +23,12 @@
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.tuples.PermutingTupleReference;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
@@ -51,17 +53,23 @@
private final List<ILSMComponent> componentsToBeMerged;
public final IModificationOperationCallback modificationCallback;
public final ISearchOperationCallback searchCallback;
+ public final PermutingTupleReference indexTuple;
+ public final MultiComparator filterCmp;
+ public final PermutingTupleReference filterTuple;
+ public ISearchPredicate searchPredicate;
public LSMRTreeOpContext(List<ILSMComponent> mutableComponents, IRTreeLeafFrame rtreeLeafFrame,
IRTreeInteriorFrame rtreeInteriorFrame, ITreeIndexFrameFactory btreeLeafFrameFactory,
ITreeIndexFrameFactory btreeInteriorFrameFactory, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback) {
+ ISearchOperationCallback searchCallback, int[] rtreeFields, int[] filterFields) {
mutableRTreeAccessors = new RTree.RTreeAccessor[mutableComponents.size()];
mutableBTreeAccessors = new BTree.BTreeAccessor[mutableComponents.size()];
rtreeOpContexts = new RTreeOpContext[mutableComponents.size()];
btreeOpContexts = new BTreeOpContext[mutableComponents.size()];
+ LSMRTreeMemoryComponent c = (LSMRTreeMemoryComponent) mutableComponents.get(0);
+
for (int i = 0; i < mutableComponents.size(); i++) {
LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) mutableComponents.get(i);
mutableRTreeAccessors[i] = (RTree.RTreeAccessor) mutableComponent.getRTree().createAccessor(
@@ -80,6 +88,16 @@
this.componentsToBeMerged = new LinkedList<ILSMComponent>();
this.modificationCallback = modificationCallback;
this.searchCallback = searchCallback;
+
+ if (filterFields != null) {
+ indexTuple = new PermutingTupleReference(rtreeFields);
+ filterCmp = MultiComparator.create(c.getLSMComponentFilter().getFilterCmpFactories());
+ filterTuple = new PermutingTupleReference(filterFields);
+ } else {
+ indexTuple = null;
+ filterCmp = null;
+ filterTuple = null;
+ }
}
public void setOperation(IndexOperation newOp) {
@@ -129,9 +147,19 @@
public IModificationOperationCallback getModificationCallback() {
return modificationCallback;
}
-
+
@Override
public List<ILSMComponent> getComponentsToBeMerged() {
return componentsToBeMerged;
}
+
+ @Override
+ public void setSearchPredicate(ISearchPredicate searchPredicate) {
+ this.searchPredicate = searchPredicate;
+ }
+
+ @Override
+ public ISearchPredicate getSearchPredicate() {
+ return searchPredicate;
+ }
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index f85a7c6..045d6cb 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -15,6 +15,7 @@
package edu.uci.ics.hyracks.storage.am.lsm.rtree.impls;
+import java.util.ArrayList;
import java.util.List;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -35,8 +36,11 @@
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.tuples.PermutingTupleReference;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
@@ -48,6 +52,7 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
@@ -66,16 +71,20 @@
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory btreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
ILSMIndexFileManager fileManager, TreeIndexFactory<RTree> diskRTreeFactory,
- TreeIndexFactory<RTree> bulkLoadRTreeFactory, IFileMapProvider diskFileMapProvider, int fieldCount,
- IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
- ILinearizeComparatorFactory linearizer, int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback) {
+ TreeIndexFactory<RTree> bulkLoadRTreeFactory, ILSMComponentFilterFactory filterFactory,
+ ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
+ IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories,
+ IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
+ int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ int[] rtreeFields, int[] filterFields) {
super(virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory,
btreeLeafFrameFactory, fileManager, new LSMRTreeWithAntiMatterTuplesDiskComponentFactory(
- diskRTreeFactory), diskFileMapProvider, fieldCount, rtreeCmpFactories, btreeCmpFactories,
- linearizer, comparatorFields, linearizerArray, 0, mergePolicy, opTracker, ioScheduler, ioOpCallback);
- bulkLoaComponentFactory = new LSMRTreeWithAntiMatterTuplesDiskComponentFactory(bulkLoadRTreeFactory);
+ diskRTreeFactory, filterFactory), diskFileMapProvider, fieldCount, rtreeCmpFactories,
+ btreeCmpFactories, linearizer, comparatorFields, linearizerArray, 0, mergePolicy, opTracker,
+ ioScheduler, ioOpCallback, filterFactory, filterFrameFactory, filterManager, rtreeFields, filterFields);
+ bulkLoaComponentFactory = new LSMRTreeWithAntiMatterTuplesDiskComponentFactory(bulkLoadRTreeFactory,
+ filterFactory);
}
@Override
@@ -179,13 +188,13 @@
// Since the LSM-RTree is used as a secondary assumption, the
// primary key will be the last comparator in the BTree comparators
- TreeTupleSorter rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getRTree().getFileId(), linearizerArray,
- rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(), flushingComponent.getRTree()
- .getBufferCache(), comparatorFields);
+ TreeTupleSorter rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getRTree().getFileId(),
+ linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(),
+ flushingComponent.getRTree().getBufferCache(), comparatorFields);
- TreeTupleSorter bTreeTupleSorter = new TreeTupleSorter(flushingComponent.getBTree().getFileId(), linearizerArray,
- btreeLeafFrameFactory.createFrame(), btreeLeafFrameFactory.createFrame(), flushingComponent.getBTree()
- .getBufferCache(), comparatorFields);
+ TreeTupleSorter bTreeTupleSorter = new TreeTupleSorter(flushingComponent.getBTree().getFileId(),
+ linearizerArray, btreeLeafFrameFactory.createFrame(), btreeLeafFrameFactory.createFrame(),
+ flushingComponent.getBTree().getBufferCache(), comparatorFields);
// BulkLoad the tuples from the in-memory tree into the new disk
// RTree.
@@ -234,6 +243,15 @@
}
rTreeBulkloader.end();
+
+ if (component.getLSMComponentFilter() != null) {
+ List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
+ filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
+ filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
+ filterManager.writeFilterInfo(component.getLSMComponentFilter(), component.getRTree());
+ }
+
return component;
}
@@ -279,6 +297,17 @@
cursor.close();
}
bulkloader.end();
+
+ if (component.getLSMComponentFilter() != null) {
+ List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
+ filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
+ filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
+ }
+ filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
+ filterManager.writeFilterInfo(component.getLSMComponentFilter(), component.getBTree());
+ }
+
return component;
}
@@ -326,6 +355,9 @@
private final IIndexBulkLoader bulkLoader;
private boolean cleanedUpArtifacts = false;
private boolean isEmptyComponent = true;
+ public final PermutingTupleReference indexTuple;
+ public final PermutingTupleReference filterTuple;
+ public final MultiComparator filterCmp;
public LSMRTreeWithAntiMatterTuplesBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
boolean checkIfEmptyIndex) throws TreeIndexException, HyracksDataException {
@@ -341,12 +373,36 @@
}
bulkLoader = ((LSMRTreeDiskComponent) component).getRTree().createBulkLoader(fillFactor, verifyInput,
numElementsHint, false);
+
+ if (filterFields != null) {
+ indexTuple = new PermutingTupleReference(rtreeFields);
+ filterCmp = MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories());
+ filterTuple = new PermutingTupleReference(filterFields);
+ } else {
+ indexTuple = null;
+ filterCmp = null;
+ filterTuple = null;
+ }
}
@Override
public void add(ITupleReference tuple) throws HyracksDataException, IndexException {
try {
- bulkLoader.add(tuple);
+ ITupleReference t;
+ if (indexTuple != null) {
+ indexTuple.reset(tuple);
+ t = indexTuple;
+ } else {
+ t = tuple;
+ }
+
+ bulkLoader.add(t);
+
+ if (filterTuple != null) {
+ filterTuple.reset(tuple);
+ component.getLSMComponentFilter().update(filterTuple, filterCmp);
+ }
+
} catch (IndexException | HyracksDataException | RuntimeException e) {
cleanupArtifacts();
throw e;
@@ -360,6 +416,12 @@
public void end() throws HyracksDataException, IndexException {
if (!cleanedUpArtifacts) {
bulkLoader.end();
+
+ if (component.getLSMComponentFilter() != null) {
+ filterManager.writeFilterInfo(component.getLSMComponentFilter(),
+ ((LSMRTreeDiskComponent) component).getRTree());
+ }
+
if (isEmptyComponent) {
cleanupArtifacts();
} else {
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java
index fe253cc..344c829 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesDiskComponentFactory.java
@@ -18,6 +18,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
@@ -25,15 +26,18 @@
public class LSMRTreeWithAntiMatterTuplesDiskComponentFactory implements ILSMComponentFactory {
private final TreeIndexFactory<RTree> rtreeFactory;
+ private final ILSMComponentFilterFactory filterFactory;
- public LSMRTreeWithAntiMatterTuplesDiskComponentFactory(TreeIndexFactory<RTree> rtreeFactory) {
+ public LSMRTreeWithAntiMatterTuplesDiskComponentFactory(TreeIndexFactory<RTree> rtreeFactory,
+ ILSMComponentFilterFactory filterFactory) {
this.rtreeFactory = rtreeFactory;
+ this.filterFactory = filterFactory;
}
@Override
public ILSMComponent createLSMComponentInstance(LSMComponentFileReferences cfr) throws IndexException {
- return new LSMRTreeDiskComponent(rtreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
- null, null);
+ return new LSMRTreeDiskComponent(rtreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()), null,
+ null, filterFactory == null ? null : filterFactory.createLSMComponentFilter());
}
@Override
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
index 224e449..2ec9107 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
@@ -33,13 +33,17 @@
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import edu.uci.ics.hyracks.storage.am.lsm.common.frames.LSMComponentFilterFrameFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BTreeFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFilterFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.ExternalRTree;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.impls.LSMRTree;
@@ -67,7 +71,9 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
- ILinearizeComparatorFactory linearizeCmpFactory, int[] buddyBTreeFields) throws TreeIndexException {
+ ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, int[] buddyBTreeFields,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields)
+ throws TreeIndexException {
ITypeTraits[] btreeTypeTraits = new ITypeTraits[buddyBTreeFields.length];
for (int i = 0; i < btreeTypeTraits.length; i++) {
@@ -107,13 +113,24 @@
BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider,
bloomFilterKeyFields);
+ LSMComponentFilterFactory filterFactory = null;
+ LSMComponentFilterFrameFactory filterFrameFactory = null;
+ LSMComponentFilterManager filterManager = null;
+ if (filterCmpFactories != null) {
+ TypeAwareTupleWriterFactory filterTupleWriterFactory = new TypeAwareTupleWriterFactory(filterTypeTraits);
+ filterFactory = new LSMComponentFilterFactory(filterTupleWriterFactory, filterCmpFactories);
+ filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory,
+ diskBufferCache.getPageSize());
+ filterManager = new LSMComponentFilterManager(diskBufferCache, filterFrameFactory);
+ }
ILSMIndexFileManager fileNameManager = new LSMRTreeFileManager(diskFileMapProvider, file, diskRTreeFactory,
diskBTreeFactory);
LSMRTree lsmTree = new LSMRTree(virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory,
btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory,
- bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length,
- rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory, comparatorFields, linearizerArray,
- mergePolicy, opTracker, ioScheduler, ioOpCallback, buddyBTreeFields);
+ bloomFilterFactory, filterFactory, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate,
+ diskFileMapProvider, typeTraits.length, rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory,
+ comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, rtreeFields,
+ buddyBTreeFields, filterFields);
return lsmTree;
}
@@ -123,7 +140,8 @@
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, ILinearizeComparatorFactory linearizerCmpFactory)
+ ILSMIOOperationCallback ioOpCallback, ILinearizeComparatorFactory linearizerCmpFactory, int[] rtreeFields,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields)
throws TreeIndexException {
LSMRTreeTupleWriterFactory rtreeTupleWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, false);
LSMRTreeTupleWriterFactory btreeTupleWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, true);
@@ -155,17 +173,39 @@
// The first field is for the sorted curve (e.g. Hilbert curve), and the
// second field is for the primary key.
- int[] comparatorFields = { 0, btreeCmpFactories.length - 1 };
- IBinaryComparatorFactory[] linearizerArray = { linearizerCmpFactory,
- btreeCmpFactories[btreeCmpFactories.length - 1] };
+ int[] comparatorFields = new int[btreeCmpFactories.length - rtreeCmpFactories.length + 1];
+ IBinaryComparatorFactory[] linearizerArray = new IBinaryComparatorFactory[btreeCmpFactories.length
+ - rtreeCmpFactories.length + 1];
+ comparatorFields[0] = 0;
+ for (int i = 1; i < comparatorFields.length; i++) {
+ comparatorFields[i] = rtreeCmpFactories.length - 1 + i;
+ }
+ linearizerArray[0] = linearizerCmpFactory;
+ int j = 1;
+ for (int i = rtreeCmpFactories.length; i < btreeCmpFactories.length; i++) {
+ linearizerArray[j] = btreeCmpFactories[i];
+ j++;
+ }
+
+ LSMComponentFilterFactory filterFactory = null;
+ LSMComponentFilterFrameFactory filterFrameFactory = null;
+ LSMComponentFilterManager filterManager = null;
+ if (filterCmpFactories != null) {
+ TypeAwareTupleWriterFactory filterTupleWriterFactory = new TypeAwareTupleWriterFactory(filterTypeTraits);
+ filterFactory = new LSMComponentFilterFactory(filterTupleWriterFactory, filterCmpFactories);
+ filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory,
+ diskBufferCache.getPageSize());
+ filterManager = new LSMComponentFilterManager(diskBufferCache, filterFrameFactory);
+ }
ILSMIndexFileManager fileNameManager = new LSMRTreeWithAntiMatterTuplesFileManager(diskFileMapProvider, file,
diskRTreeFactory);
LSMRTreeWithAntiMatterTuples lsmTree = new LSMRTreeWithAntiMatterTuples(virtualBufferCaches,
rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory,
- fileNameManager, diskRTreeFactory, bulkLoadRTreeFactory, diskFileMapProvider, typeTraits.length,
- rtreeCmpFactories, btreeCmpFactories, linearizerCmpFactory, comparatorFields, linearizerArray,
- mergePolicy, opTracker, ioScheduler, ioOpCallback);
+ fileNameManager, diskRTreeFactory, bulkLoadRTreeFactory, filterFactory, filterFrameFactory,
+ filterManager, diskFileMapProvider, typeTraits.length, rtreeCmpFactories, btreeCmpFactories,
+ linearizerCmpFactory, comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler,
+ ioOpCallback, rtreeFields, filterFields);
return lsmTree;
}
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorDescriptor.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorDescriptor.java
index 4498e6c..e516478 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorDescriptor.java
@@ -37,23 +37,30 @@
private static final long serialVersionUID = 1L;
protected int[] keyFields; // fields in input tuple to be used as keys
+ protected final int[] minFilterFieldIndexes;
+ protected final int[] maxFilterFieldIndexes;
public RTreeSearchOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc,
IStorageManagerInterface storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider,
IFileSplitProvider fileSplitProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] comparatorFactories, int[] keyFields,
IIndexDataflowHelperFactory dataflowHelperFactory, boolean retainInput, boolean retainNull,
- INullWriterFactory nullWriterFactory, ISearchOperationCallbackFactory searchOpCallbackFactory) {
+ INullWriterFactory nullWriterFactory, ISearchOperationCallbackFactory searchOpCallbackFactory,
+ int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes) {
+
super(spec, 1, 1, recDesc, storageManager, lifecycleManagerProvider, fileSplitProvider, typeTraits,
comparatorFactories, null, dataflowHelperFactory, null, retainInput, retainNull, nullWriterFactory,
NoOpLocalResourceFactoryProvider.INSTANCE, searchOpCallbackFactory,
NoOpOperationCallbackFactory.INSTANCE);
this.keyFields = keyFields;
+ this.minFilterFieldIndexes = minFilterFieldIndexes;
+ this.maxFilterFieldIndexes = maxFilterFieldIndexes;
}
@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
- return new RTreeSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, keyFields);
+ return new RTreeSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, keyFields,
+ minFilterFieldIndexes, maxFilterFieldIndexes);
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
index acb3ece..77efc2e 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
@@ -31,8 +31,9 @@
protected MultiComparator cmp;
public RTreeSearchOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition, IRecordDescriptorProvider recordDescProvider, int[] keyFields) {
- super(opDesc, ctx, partition, recordDescProvider);
+ int partition, IRecordDescriptorProvider recordDescProvider, int[] keyFields, int[] minFilterFieldIndexes,
+ int[] maxFilterFieldIndexes) {
+ super(opDesc, ctx, partition, recordDescProvider, minFilterFieldIndexes, maxFilterFieldIndexes);
if (keyFields != null && keyFields.length > 0) {
searchKey = new PermutingFrameTupleReference();
searchKey.setFieldPermutation(keyFields);
@@ -43,7 +44,7 @@
protected ISearchPredicate createSearchPredicate() {
ITreeIndex treeIndex = (ITreeIndex) index;
cmp = RTreeUtils.getSearchMultiComparator(treeIndex.getComparatorFactories(), searchKey);
- return new SearchPredicate(searchKey, cmp);
+ return new SearchPredicate(searchKey, cmp, minFilterKey, maxFilterKey);
}
@Override
@@ -51,6 +52,12 @@
if (searchKey != null) {
searchKey.reset(accessor, tupleIndex);
}
+ if (minFilterKey != null) {
+ minFilterKey.reset(accessor, tupleIndex);
+ }
+ if (maxFilterKey != null) {
+ maxFilterKey.reset(accessor, tupleIndex);
+ }
}
@Override
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/SearchPredicate.java b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/SearchPredicate.java
index 2d621f3..35586e1 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/SearchPredicate.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/SearchPredicate.java
@@ -16,34 +16,41 @@
package edu.uci.ics.hyracks.storage.am.rtree.impls;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
+import edu.uci.ics.hyracks.storage.am.common.impls.AbstractSearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-public class SearchPredicate implements ISearchPredicate {
+public class SearchPredicate extends AbstractSearchPredicate {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 1L;
- protected ITupleReference searchKey;
- protected MultiComparator cmp;
+ protected ITupleReference searchKey;
+ protected MultiComparator cmp;
- public SearchPredicate(ITupleReference searchKey, MultiComparator cmp) {
- this.searchKey = searchKey;
- this.cmp = cmp;
- }
+ public SearchPredicate(ITupleReference searchKey, MultiComparator cmp) {
+ this.searchKey = searchKey;
+ this.cmp = cmp;
+ }
- public ITupleReference getSearchKey() {
- return searchKey;
- }
+ public SearchPredicate(ITupleReference searchKey, MultiComparator cmp, ITupleReference minFilterTuple,
+ ITupleReference maxFilterTuple) {
+ super(minFilterTuple, maxFilterTuple);
+ this.searchKey = searchKey;
+ this.cmp = cmp;
+ }
- public void setSearchKey(ITupleReference searchKey) {
- this.searchKey = searchKey;
- }
+ public ITupleReference getSearchKey() {
+ return searchKey;
+ }
- public MultiComparator getLowKeyComparator() {
- return cmp;
- }
+ public void setSearchKey(ITupleReference searchKey) {
+ this.searchKey = searchKey;
+ }
- public MultiComparator getHighKeyComparator() {
- return cmp;
- }
+ public MultiComparator getLowKeyComparator() {
+ return cmp;
+ }
+
+ public MultiComparator getHighKeyComparator() {
+ return cmp;
+ }
}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexExamplesTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
index f814829..1ba804d 100644
--- a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
@@ -56,7 +56,8 @@
protected final Random rnd = new Random(50);
protected abstract ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
- int[] bloomFilterKeyFields) throws TreeIndexException;
+ int[] bloomFilterKeyFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
+ int[] btreeFields, int[] filterFields) throws TreeIndexException;
/**
* Fixed-Length Key,Value Example. Create a tree index with one fixed-length
@@ -87,7 +88,7 @@
int[] bloomFilterKeyFields = new int[keyFieldCount];
bloomFilterKeyFields[0] = 0;
- ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields);
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
treeIndex.create();
treeIndex.activate();
@@ -132,7 +133,7 @@
ArrayTupleReference highKey = new ArrayTupleReference();
TupleUtils.createIntegerTuple(highKeyTb, highKey, 1000);
- rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey);
+ rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey, null, null);
treeIndex.validate();
treeIndex.deactivate();
@@ -171,7 +172,7 @@
int[] bloomFilterKeyFields = new int[keyFieldCount];
bloomFilterKeyFields[0] = 0;
- ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields);
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
treeIndex.create();
treeIndex.activate();
@@ -248,7 +249,7 @@
bloomFilterKeyFields[0] = 0;
bloomFilterKeyFields[1] = 1;
- ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields);
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
treeIndex.create();
treeIndex.activate();
@@ -295,7 +296,7 @@
TupleUtils.createIntegerTuple(highKeyTb, highKey, 3);
// Prefix-Range search in [-3, 3]
- rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey);
+ rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey, null, null);
treeIndex.validate();
treeIndex.deactivate();
@@ -331,7 +332,7 @@
int[] bloomFilterKeyFields = new int[keyFieldCount];
bloomFilterKeyFields[0] = 0;
- ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields);
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
treeIndex.create();
treeIndex.activate();
@@ -378,7 +379,7 @@
ArrayTupleReference highKey = new ArrayTupleReference();
TupleUtils.createTuple(highKeyTb, highKey, fieldSerdes, "cc7");
- rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey);
+ rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey, null, null);
treeIndex.validate();
treeIndex.deactivate();
@@ -415,7 +416,7 @@
int[] bloomFilterKeyFields = new int[keyFieldCount];
bloomFilterKeyFields[0] = 0;
- ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields);
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
treeIndex.create();
treeIndex.activate();
@@ -521,7 +522,7 @@
int[] bloomFilterKeyFields = new int[keyFieldCount];
bloomFilterKeyFields[0] = 0;
- ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields);
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
treeIndex.create();
treeIndex.activate();
@@ -612,7 +613,7 @@
bloomFilterKeyFields[0] = 0;
bloomFilterKeyFields[1] = 1;
- ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields);
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
treeIndex.create();
treeIndex.activate();
@@ -649,7 +650,7 @@
TupleUtils.createIntegerTuple(highKeyTb, highKey, 44500);
// Prefix-Range search in [44444, 44500]
- rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey);
+ rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey, null, null);
treeIndex.validate();
treeIndex.deactivate();
@@ -687,7 +688,8 @@
int ins = 1000;
for (int i = 1; i < ins; i++) {
- ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields);
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null,
+ null);
treeIndex.create();
treeIndex.activate();
@@ -731,7 +733,7 @@
}
}
- private void orderedScan(IIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes) throws Exception {
+ protected void orderedScan(IIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes) throws Exception {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Ordered Scan:");
}
@@ -752,7 +754,7 @@
}
}
- private void diskOrderScan(IIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes) throws Exception {
+ protected void diskOrderScan(IIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes) throws Exception {
try {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Disk-Order Scan:");
@@ -788,8 +790,9 @@
}
}
- private void rangeSearch(IBinaryComparatorFactory[] cmpFactories, IIndexAccessor indexAccessor,
- ISerializerDeserializer[] fieldSerdes, ITupleReference lowKey, ITupleReference highKey) throws Exception {
+ protected void rangeSearch(IBinaryComparatorFactory[] cmpFactories, IIndexAccessor indexAccessor,
+ ISerializerDeserializer[] fieldSerdes, ITupleReference lowKey, ITupleReference highKey,
+ ITupleReference minFilterTuple, ITupleReference maxFilterTuple) throws Exception {
if (LOGGER.isLoggable(Level.INFO)) {
String lowKeyString = TupleUtils.printTuple(lowKey, fieldSerdes);
String highKeyString = TupleUtils.printTuple(highKey, fieldSerdes);
@@ -798,7 +801,13 @@
ITreeIndexCursor rangeCursor = (ITreeIndexCursor) indexAccessor.createSearchCursor(false);
MultiComparator lowKeySearchCmp = BTreeUtils.getSearchMultiComparator(cmpFactories, lowKey);
MultiComparator highKeySearchCmp = BTreeUtils.getSearchMultiComparator(cmpFactories, highKey);
- RangePredicate rangePred = new RangePredicate(lowKey, highKey, true, true, lowKeySearchCmp, highKeySearchCmp);
+ RangePredicate rangePred;
+ if (minFilterTuple != null && maxFilterTuple != null) {
+ rangePred = new RangePredicate(lowKey, highKey, true, true, lowKeySearchCmp, highKeySearchCmp,
+ minFilterTuple, maxFilterTuple);
+ } else {
+ rangePred = new RangePredicate(lowKey, highKey, true, true, lowKeySearchCmp, highKeySearchCmp);
+ }
indexAccessor.search(rangeCursor, rangePred);
try {
while (rangeCursor.hasNext()) {
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
index ff3d21c..a693896 100644
--- a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
@@ -65,8 +65,9 @@
protected abstract ITreeIndex createTreeIndex(ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
- IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType, int[] btreeFields)
- throws TreeIndexException;
+ IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
+ int[] rtreeFields, int[] btreeFields, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) throws TreeIndexException;
/**
* Two Dimensions Example. Create an RTree index of two dimensions, where
@@ -134,7 +135,7 @@
rtreeCmpFactories.length, IntegerPointable.FACTORY);
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, RTreePolicyType.RTREE, btreeFields);
+ valueProviderFactories, RTreePolicyType.RTREE, null, btreeFields, null, null, null);
treeIndex.create();
treeIndex.activate();
@@ -176,7 +177,7 @@
ArrayTupleReference key = new ArrayTupleReference();
TupleUtils.createIntegerTuple(keyTb, key, -1000, -1000, 1000, 1000);
- rangeSearch(rtreeCmpFactories, indexAccessor, fieldSerdes, key);
+ rangeSearch(rtreeCmpFactories, indexAccessor, fieldSerdes, key, null, null);
treeIndex.deactivate();
treeIndex.destroy();
@@ -245,7 +246,7 @@
//2
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, RTreePolicyType.RTREE, btreeFields);
+ valueProviderFactories, RTreePolicyType.RTREE, null, btreeFields, null, null, null);
treeIndex.create();
treeIndex.activate();
@@ -384,7 +385,7 @@
rtreeCmpFactories.length, IntegerPointable.FACTORY);
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, RTreePolicyType.RSTARTREE, btreeFields);
+ valueProviderFactories, RTreePolicyType.RSTARTREE, null, btreeFields, null, null, null);
treeIndex.create();
treeIndex.activate();
@@ -534,7 +535,7 @@
//4
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, RTreePolicyType.RTREE, btreeFields);
+ valueProviderFactories, RTreePolicyType.RTREE, null, btreeFields, null, null, null);
treeIndex.create();
treeIndex.activate();
@@ -577,7 +578,7 @@
ArrayTupleReference key = new ArrayTupleReference();
TupleUtils.createDoubleTuple(keyTb, key, -1000.0, -1000.0, -1000.0, 1000.0, 1000.0, 1000.0);
- rangeSearch(rtreeCmpFactories, indexAccessor, fieldSerdes, key);
+ rangeSearch(rtreeCmpFactories, indexAccessor, fieldSerdes, key, null, null);
treeIndex.deactivate();
treeIndex.destroy();
@@ -641,7 +642,7 @@
rtreeCmpFactories.length, IntegerPointable.FACTORY);
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, RTreePolicyType.RTREE, btreeFields);
+ valueProviderFactories, RTreePolicyType.RTREE, null, btreeFields, null, null, null);
treeIndex.create();
treeIndex.activate();
@@ -778,7 +779,7 @@
//6
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, RTreePolicyType.RTREE, btreeFields);
+ valueProviderFactories, RTreePolicyType.RTREE, null, btreeFields, null, null, null);
treeIndex.create();
treeIndex.activate();
@@ -819,13 +820,13 @@
ArrayTupleReference key = new ArrayTupleReference();
TupleUtils.createIntegerTuple(keyTb, key, -1000, -1000, 1000, 1000);
- rangeSearch(rtreeCmpFactories, indexAccessor, fieldSerdes, key);
+ rangeSearch(rtreeCmpFactories, indexAccessor, fieldSerdes, key, null, null);
treeIndex.deactivate();
treeIndex.destroy();
}
- private void scan(IIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes) throws Exception {
+ protected void scan(IIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes) throws Exception {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Scan:");
}
@@ -846,7 +847,7 @@
}
}
- private void diskOrderScan(IIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes) throws Exception {
+ protected void diskOrderScan(IIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes) throws Exception {
try {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Disk-Order Scan:");
@@ -882,15 +883,23 @@
}
}
- private void rangeSearch(IBinaryComparatorFactory[] cmpFactories, IIndexAccessor indexAccessor,
- ISerializerDeserializer[] fieldSerdes, ITupleReference key) throws Exception {
+ protected void rangeSearch(IBinaryComparatorFactory[] cmpFactories, IIndexAccessor indexAccessor,
+ ISerializerDeserializer[] fieldSerdes, ITupleReference key, ITupleReference minFilterTuple,
+ ITupleReference maxFilterTuple) throws Exception {
if (LOGGER.isLoggable(Level.INFO)) {
String kString = TupleUtils.printTuple(key, fieldSerdes);
LOGGER.info("Range-Search using key: " + kString);
}
ITreeIndexCursor rangeCursor = (ITreeIndexCursor) indexAccessor.createSearchCursor(false);
MultiComparator cmp = RTreeUtils.getSearchMultiComparator(cmpFactories, key);
- SearchPredicate rangePred = new SearchPredicate(key, cmp);
+
+ SearchPredicate rangePred;
+ if (minFilterTuple != null && maxFilterTuple != null) {
+ rangePred = new SearchPredicate(key, cmp, minFilterTuple, maxFilterTuple);
+ } else {
+ rangePred = new SearchPredicate(key, cmp);
+ }
+
indexAccessor.search(rangeCursor, rangePred);
try {
while (rangeCursor.hasNext()) {
@@ -905,5 +914,4 @@
rangeCursor.close();
}
}
-
}
\ No newline at end of file
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java
index 0405b3b..d9459f6 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java
@@ -41,7 +41,8 @@
}
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
- int[] bloomFilterKeyFields) throws TreeIndexException {
+ int[] bloomFilterKeyFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
+ int[] btreeFields, int[] filterFields) throws TreeIndexException {
return BTreeUtils.createBTree(harness.getBufferCache(), harness.getFileMapProvider(), typeTraits, cmpFactories,
BTreeLeafFrameType.REGULAR_NSM, harness.getFileReference());
}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
index f174ce4..7e535d8 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
@@ -15,14 +15,26 @@
package edu.uci.ics.hyracks.storage.am.lsm.btree;
+import java.util.logging.Level;
+
import org.junit.After;
import org.junit.Before;
+import org.junit.Test;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
import edu.uci.ics.hyracks.storage.am.btree.OrderedIndexExamplesTest;
+import edu.uci.ics.hyracks.storage.am.common.TestOperationCallback;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
@@ -33,11 +45,13 @@
@Override
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
- int[] bloomFilterKeyFields) throws TreeIndexException {
+ int[] bloomFilterKeyFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
+ int[] btreeFields, int[] filterFields) throws TreeIndexException {
return LSMBTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, cmpFactories,
bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), true);
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), true,
+ filterTypeTraits, filterCmpFactories, btreeFields, filterFields);
}
@Before
@@ -50,4 +64,97 @@
harness.tearDown();
}
+ /**
+ * Test the LSM component filters.
+ */
+ @Test
+ public void additionalFilteringingExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Testing LSMBTree component filters.");
+ }
+
+ // Declare fields.
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE };
+
+ // Declare keys.
+ int keyFieldCount = 1;
+ IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
+ cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ // This is only used for the LSM-BTree.
+ int[] bloomFilterKeyFields = new int[keyFieldCount];
+ bloomFilterKeyFields[0] = 0;
+
+ ITypeTraits[] filterTypeTraits = { IntegerPointable.TYPE_TRAITS };
+ IBinaryComparatorFactory[] filterCmpFactories = { PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) };
+ int[] filterFields = { 1 };
+ int[] btreeFields = { 1 };
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, filterTypeTraits,
+ filterCmpFactories, btreeFields, filterFields);
+ treeIndex.create();
+ treeIndex.activate();
+
+ long start = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inserting into tree...");
+ }
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor(TestOperationCallback.INSTANCE,
+ TestOperationCallback.INSTANCE);
+ int numInserts = 10000;
+ for (int i = 0; i < numInserts; i++) {
+ int f0 = rnd.nextInt() % numInserts;
+ int f1 = i;
+ TupleUtils.createIntegerTuple(tb, tuple, f0, f1);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + i + " : " + f0 + " " + f1);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
+ }
+
+ orderedScan(indexAccessor, fieldSerdes);
+ diskOrderScan(indexAccessor, fieldSerdes);
+
+ // Build low key.
+ ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(keyFieldCount);
+ ArrayTupleReference lowKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(lowKeyTb, lowKey, -1000);
+
+ // Build high key.
+ ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(keyFieldCount);
+ ArrayTupleReference highKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(highKeyTb, highKey, 1000);
+
+ // Build min filter key.
+ ArrayTupleBuilder minFilterTb = new ArrayTupleBuilder(filterFields.length);
+ ArrayTupleReference minTuple = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(minFilterTb, minTuple, 400);
+
+ // Build max filter key.
+ ArrayTupleBuilder maxFilterTb = new ArrayTupleBuilder(filterFields.length);
+ ArrayTupleReference maxTuple = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(maxFilterTb, maxTuple, 500);
+
+ rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey, minTuple, maxTuple);
+
+ treeIndex.validate();
+ treeIndex.deactivate();
+ treeIndex.destroy();
+ }
}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
index 9881975..6f5afc3 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
@@ -48,7 +48,7 @@
SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), bloomFilterKeyFields,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
NoOpOperationTrackerProvider.INSTANCE.getOperationTracker(null), harness.getIOScheduler(),
- harness.getIOOperationCallback(), true);
+ harness.getIOOperationCallback(), true, null, null, null, null);
}
@Override
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
index d4a0857..a4dd7b2 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
@@ -55,7 +55,7 @@
SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), bloomFilterKeyFields,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
NoOpOperationTrackerProvider.INSTANCE.getOperationTracker(null), harness.getIOScheduler(),
- harness.getIOOperationCallback(), true);
+ harness.getIOOperationCallback(), true, null, null, null, null);
}
@Override
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
index c6f0a65..a41e112 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
@@ -53,7 +53,8 @@
return LSMBTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, cmpFactories,
bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), true);
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), true, null,
+ null, null, null);
}
@Override
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
index 045aed0..bbe770b 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
@@ -109,7 +109,8 @@
lsmtree = LSMBTreeUtils.createLSMTree(virtualBufferCaches, file, bufferCache, fmp, typeTraits, cmpFactories,
bloomFilterKeyFields, bloomFilterFalsePositiveRate, NoMergePolicy.INSTANCE,
- new ThreadCountingTracker(), ioScheduler, NoOpIOOperationCallback.INSTANCE, true);
+ new ThreadCountingTracker(), ioScheduler, NoOpIOOperationCallback.INSTANCE, true, null, null, null,
+ null);
}
@Override
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
index 84ea6df..beecd7b 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
@@ -76,7 +76,7 @@
}
LSMBTree lsmTree = LSMBTreeUtils.createLSMTree(virtualBufferCaches, file, diskBufferCache, diskFileMapProvider,
typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
- ioScheduler, ioOpCallback, true);
+ ioScheduler, ioOpCallback, true, null, null, null, null);
LSMBTreeTestContext testCtx = new LSMBTreeTestContext(fieldSerdes, lsmTree);
return testCtx;
}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
index 059df73..cf18a80 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
@@ -98,7 +98,9 @@
public static LSMInvertedIndexTestContext create(LSMInvertedIndexTestHarness harness,
ISerializerDeserializer[] fieldSerdes, int tokenFieldCount, IBinaryTokenizerFactory tokenizerFactory,
- InvertedIndexType invIndexType) throws IndexException, HyracksDataException {
+ InvertedIndexType invIndexType, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps,
+ int[] invertedIndexFieldsForNonBulkLoadOps) throws IndexException, HyracksDataException {
ITypeTraits[] allTypeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] allCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes,
fieldSerdes.length);
@@ -152,7 +154,9 @@
harness.getDiskFileMapProvider(), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, tokenizerFactory, harness.getDiskBufferCache(), harness.getOnDiskDir(),
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback());
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
+ invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields,
+ filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps);
break;
}
case PARTITIONED_LSM: {
@@ -160,7 +164,9 @@
harness.getDiskFileMapProvider(), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
tokenCmpFactories, tokenizerFactory, harness.getDiskBufferCache(), harness.getOnDiskDir(),
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback());
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(),
+ invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields,
+ filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps);
break;
}
default: {
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
index 58e0ace..ee98a99 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
@@ -157,7 +157,7 @@
IBinaryTokenizerFactory tokenizerFactory = new DelimitedUTF8StringBinaryTokenizerFactory(true, false,
tokenFactory);
LSMInvertedIndexTestContext testCtx = LSMInvertedIndexTestContext.create(harness, fieldSerdes,
- fieldSerdes.length - 1, tokenizerFactory, invIndexType);
+ fieldSerdes.length - 1, tokenizerFactory, invIndexType, null, null, null, null, null, null);
return testCtx;
}
@@ -168,7 +168,7 @@
IBinaryTokenizerFactory tokenizerFactory = new DelimitedUTF8StringBinaryTokenizerFactory(true, false,
tokenFactory);
LSMInvertedIndexTestContext testCtx = LSMInvertedIndexTestContext.create(harness, fieldSerdes,
- fieldSerdes.length - 1, tokenizerFactory, invIndexType);
+ fieldSerdes.length - 1, tokenizerFactory, invIndexType, null, null, null, null, null, null);
return testCtx;
}
@@ -179,7 +179,7 @@
IBinaryTokenizerFactory tokenizerFactory = new NGramUTF8StringBinaryTokenizerFactory(TEST_GRAM_LENGTH, true,
true, false, tokenFactory);
LSMInvertedIndexTestContext testCtx = LSMInvertedIndexTestContext.create(harness, fieldSerdes,
- fieldSerdes.length - 1, tokenizerFactory, invIndexType);
+ fieldSerdes.length - 1, tokenizerFactory, invIndexType, null, null, null, null, null, null);
return testCtx;
}
@@ -190,7 +190,7 @@
IBinaryTokenizerFactory tokenizerFactory = new NGramUTF8StringBinaryTokenizerFactory(TEST_GRAM_LENGTH, true,
true, false, tokenFactory);
LSMInvertedIndexTestContext testCtx = LSMInvertedIndexTestContext.create(harness, fieldSerdes,
- fieldSerdes.length - 1, tokenizerFactory, invIndexType);
+ fieldSerdes.length - 1, tokenizerFactory, invIndexType, null, null, null, null, null, null);
return testCtx;
}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/AbstractLSMRTreeExamplesTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/AbstractLSMRTreeExamplesTest.java
new file mode 100644
index 0000000..028c194
--- /dev/null
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/AbstractLSMRTreeExamplesTest.java
@@ -0,0 +1,168 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.rtree;
+
+import java.util.logging.Level;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeExamplesTest;
+import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
+
+public abstract class AbstractLSMRTreeExamplesTest extends AbstractRTreeExamplesTest {
+
+ /**
+ * Test the LSM component filters.
+ */
+ @Test
+ public void additionalFilteringingExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Testing LSMRTree or LSMRTreeWithAntiMatterTuples component filters.");
+ }
+
+ // Declare fields.
+ int fieldCount = 6;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[3] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[4] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[5] = IntegerPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE };
+
+ // Declare RTree keys.
+ int rtreeKeyFieldCount = 4;
+ IBinaryComparatorFactory[] rtreeCmpFactories = new IBinaryComparatorFactory[rtreeKeyFieldCount];
+ rtreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ rtreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ rtreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ // Declare BTree keys, this will only be used for LSMRTree
+ int btreeKeyFieldCount;
+ IBinaryComparatorFactory[] btreeCmpFactories;
+ int[] btreeFields = null;
+ if (rTreeType == RTreeType.LSMRTREE) {
+ //Parameters look different for LSM RTREE from LSM RTREE WITH ANTI MATTER TUPLES
+ btreeKeyFieldCount = 1;
+ btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeFields = new int[btreeKeyFieldCount];
+ for (int i = 0; i < btreeKeyFieldCount; i++) {
+ btreeFields[i] = rtreeKeyFieldCount + i;
+ }
+
+ } else {
+ btreeKeyFieldCount = 6;
+ btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[5] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ }
+
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ rtreeCmpFactories.length, IntegerPointable.FACTORY);
+
+ int[] rtreeFields = { 0, 1, 2, 3, 4 };
+ ITypeTraits[] filterTypeTraits = { IntegerPointable.TYPE_TRAITS };
+ IBinaryComparatorFactory[] filterCmpFactories = { PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) };
+ int[] filterFields = { 5 };
+
+ ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
+ valueProviderFactories, RTreePolicyType.RTREE, rtreeFields, btreeFields, filterTypeTraits,
+ filterCmpFactories, filterFields);
+ treeIndex.create();
+ treeIndex.activate();
+
+ long start = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inserting into tree...");
+ }
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ IIndexAccessor indexAccessor = (IIndexAccessor) treeIndex.createAccessor(NoOpOperationCallback.INSTANCE,
+ NoOpOperationCallback.INSTANCE);
+ int numInserts = 10000;
+ for (int i = 0; i < numInserts; i++) {
+ int p1x = rnd.nextInt();
+ int p1y = rnd.nextInt();
+ int p2x = rnd.nextInt();
+ int p2y = rnd.nextInt();
+
+ int pk = 5;
+ int filter = i;
+
+ TupleUtils.createIntegerTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x),
+ Math.max(p1y, p2y), pk, filter);
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
+ }
+
+ scan(indexAccessor, fieldSerdes);
+ diskOrderScan(indexAccessor, fieldSerdes);
+
+ // Build key.
+ ArrayTupleBuilder keyTb = new ArrayTupleBuilder(rtreeKeyFieldCount);
+ ArrayTupleReference key = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(keyTb, key, -1000, -1000, 1000, 1000);
+
+ // Build min filter key.
+ ArrayTupleBuilder minFilterTb = new ArrayTupleBuilder(filterFields.length);
+ ArrayTupleReference minTuple = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(minFilterTb, minTuple, 400);
+
+ // Build max filter key.
+ ArrayTupleBuilder maxFilterTb = new ArrayTupleBuilder(filterFields.length);
+ ArrayTupleReference maxTuple = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(maxFilterTb, maxTuple, 500);
+
+ rangeSearch(rtreeCmpFactories, indexAccessor, fieldSerdes, key, minTuple, maxTuple);
+
+ treeIndex.deactivate();
+ treeIndex.destroy();
+ }
+
+}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
index 2b02656..29f6218 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
@@ -27,10 +27,9 @@
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestHarness;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.utils.LSMRTreeUtils;
-import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeExamplesTest;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
-public class LSMRTreeExamplesTest extends AbstractRTreeExamplesTest {
+public class LSMRTreeExamplesTest extends AbstractLSMRTreeExamplesTest {
private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
public LSMRTreeExamplesTest() {
@@ -41,13 +40,15 @@
@Override
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType, int[] btreeFields) throws TreeIndexException {
+ RTreePolicyType rtreePolicyType, int[] rtreeFields, int[] btreeFields, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) throws TreeIndexException {
return LSMRTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, rtreeCmpFactories,
btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
harness.getIOOperationCallback(),
- LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), btreeFields);
+ LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), rtreeFields, btreeFields,
+ filterTypeTraits, filterCmpFactories, filterFields);
}
@Before
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
index 3ce30c5..13b256e 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
@@ -27,10 +27,9 @@
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestHarness;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.utils.LSMRTreeUtils;
-import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeExamplesTest;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
-public class LSMRTreeWithAntiMatterTuplesExamplesTest extends AbstractRTreeExamplesTest {
+public class LSMRTreeWithAntiMatterTuplesExamplesTest extends AbstractLSMRTreeExamplesTest {
private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
public LSMRTreeWithAntiMatterTuplesExamplesTest() {
@@ -41,13 +40,15 @@
@Override
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType, int[] btreeFields) throws TreeIndexException {
+ RTreePolicyType rtreePolicyType, int[] rtreeFields, int[] btreeFields, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) throws TreeIndexException {
return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits,
rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
harness.getIOOperationCallback(),
- LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length));
+ LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), rtreeFields,
+ filterTypeTraits, filterCmpFactories, filterFields);
}
@Before
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
index c3ec1b3..08a1e1c 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
@@ -63,7 +63,8 @@
btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
harness.getIOOperationCallback(),
- LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), btreeFields);
+ LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), null, btreeFields, null,
+ null, null);
}
@Override
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
index 22a4b07..10d8fe2 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
@@ -63,7 +63,7 @@
rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
harness.getIOOperationCallback(),
- LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length));
+ LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), null, null, null, null);
}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
index 484ebc6..8966dc9 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
@@ -87,7 +87,8 @@
LSMRTree lsmTree = LSMRTreeUtils.createLSMTree(virtualBufferCaches, file, diskBufferCache, diskFileMapProvider,
typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback,
- LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), btreeFields);
+ LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), null, btreeFields, null,
+ null, null);
LSMRTreeTestContext testCtx = new LSMRTreeTestContext(fieldSerdes, lsmTree);
return testCtx;
}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
index 6a0a6bb..e1483b3 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeWithAntiMatterTuplesTestContext.java
@@ -82,7 +82,7 @@
LSMRTreeWithAntiMatterTuples lsmTree = LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(virtualBufferCaches,
file, diskBufferCache, diskFileMapProvider, typeTraits, rtreeCmpFactories, btreeCmpFactories,
valueProviderFactories, rtreePolicyType, mergePolicy, opTracker, ioScheduler, ioOpCallback,
- LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length));
+ LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), null, null, null, null);
LSMRTreeWithAntiMatterTuplesTestContext testCtx = new LSMRTreeWithAntiMatterTuplesTestContext(fieldSerdes,
lsmTree);
return testCtx;
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeExamplesTest.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeExamplesTest.java
index 8bec6d9..15450e3 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeExamplesTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeExamplesTest.java
@@ -49,7 +49,8 @@
@Override
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType, int[] btreeFields) throws TreeIndexException {
+ RTreePolicyType rtreePolicyType, int[] rtreeFields, int[] btreeFields, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) throws TreeIndexException {
return RTreeUtils.createRTree(harness.getBufferCache(), harness.getFileMapProvider(), typeTraits,
valueProviderFactories, rtreeCmpFactories, rtreePolicyType, harness.getFileReference());
}
diff --git a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGen.java b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGen.java
index adc495e..0c96324 100644
--- a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGen.java
+++ b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGen.java
@@ -368,7 +368,7 @@
BTreeSearchOperatorDescriptor scanner = new BTreeSearchOperatorDescriptor(spec, recordDescriptor,
storageManagerInterface, lcManagerProvider, fileSplitProvider, typeTraits, comparatorFactories, null,
null, null, true, true, getIndexDataflowHelperFactory(), false, false, null,
- NoOpOperationCallbackFactory.INSTANCE);
+ NoOpOperationCallbackFactory.INSTANCE, null, null);
setLocationConstraint(spec, scanner);
/**
@@ -515,7 +515,8 @@
return new LSMBTreeDataflowHelperFactory(new VirtualBufferCacheProvider(),
new ConstantMergePolicyFactory(), MERGE_POLICY_PROPERTIES, NoOpOperationTrackerProvider.INSTANCE,
/* TODO verify whether key dup check is required or not in preglix: to be safe, just check it as it has been done*/
- SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE, 0.01, true);
+ SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE, 0.01, true, null, null,
+ null, null);
} else {
return new BTreeDataflowHelperFactory();
}
@@ -628,7 +629,7 @@
BTreeSearchOperatorDescriptor scanner = new BTreeSearchOperatorDescriptor(spec, recordDescriptor,
storageManagerInterface, lcManagerProvider, fileSplitProvider, typeTraits, comparatorFactories, null,
null, null, true, true, getIndexDataflowHelperFactory(), false, false, null,
- NoOpOperationCallbackFactory.INSTANCE);
+ NoOpOperationCallbackFactory.INSTANCE, null, null);
setLocationConstraint(spec, scanner);
ExternalSortOperatorDescriptor sort = null;
diff --git a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenInnerJoin.java b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenInnerJoin.java
index 8122648..4e8d3a2 100644
--- a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenInnerJoin.java
+++ b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenInnerJoin.java
@@ -632,8 +632,8 @@
BTreeSearchOperatorDescriptor scanner = new BTreeSearchOperatorDescriptor(spec, recordDescriptor,
storageManagerInterface, lcManagerProvider, secondaryFileSplitProviderRead, typeTraits,
- comparatorFactories, null, null, null, true, true, getIndexDataflowHelperFactory(), false,
- false, null, NoOpOperationCallbackFactory.INSTANCE);
+ comparatorFactories, null, null, null, true, true, getIndexDataflowHelperFactory(), false, false, null,
+ NoOpOperationCallbackFactory.INSTANCE, null, null);
setLocationConstraint(spec, scanner);
/**