Secondary RTree indexes can now be created on nullable fields (nulls are filtered during load). Still need to modify insert/delete jobgen to filter nulls as well.
git-svn-id: https://asterixdb.googlecode.com/svn/branches/asterix-fix-issue-9@214 eaa15691-b419-025a-1212-ee371bd00084
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceBTreeIndexSearchRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceBTreeIndexSearchRule.java
index 43aeafd..63b3b3b 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceBTreeIndexSearchRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceBTreeIndexSearchRule.java
@@ -621,7 +621,8 @@
ARecordType itemType) throws AlgebricksException {
List<Object> types = new ArrayList<Object>();
for (String sk : acid.getFieldExprs()) {
- types.add(AqlCompiledIndexDecl.keyFieldType(sk, itemType));
+ Pair<IAType, Boolean> keyPair = AqlCompiledIndexDecl.getNonNullableKeyFieldType(sk, itemType);
+ types.add(keyPair.first);
}
for (Triple<IEvaluatorFactory, ScalarFunctionCallExpression, IAType> t : DatasetUtils
.getPartitioningFunctions(ddecl)) {
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceRTreeIndexSearchRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceRTreeIndexSearchRule.java
index 6b76938..589baf4 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceRTreeIndexSearchRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceRTreeIndexSearchRule.java
@@ -178,8 +178,8 @@
foundVar = findIdxExprs(adecl, primIdxFields, primIdxDecl, foundIdxExprs, outComparedVars, var, fieldName);
if (foundVar) {
- IAType spatialType = AqlCompiledIndexDecl.keyFieldType(fieldName, recordType);
- dimension = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
+ Pair<IAType, Boolean> spatialTypePair = AqlCompiledIndexDecl.getNonNullableKeyFieldType(fieldName, recordType);
+ dimension = NonTaggedFormatUtil.getNumDimensions(spatialTypePair.first.getTypeTag());
}
fldPos++;
@@ -380,8 +380,8 @@
private static List<Object> secondaryIndexTypes(AqlCompiledDatasetDecl ddecl, AqlCompiledIndexDecl acid,
ARecordType itemType, int numKeys) throws AlgebricksException {
List<Object> types = new ArrayList<Object>();
- IAType keyType = AqlCompiledIndexDecl.keyFieldType(acid.getFieldExprs().get(0), itemType);
- IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(keyType.getTypeTag());
+ Pair<IAType, Boolean> keyTypePair = AqlCompiledIndexDecl.getNonNullableKeyFieldType(acid.getFieldExprs().get(0), itemType);
+ IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(keyTypePair.first.getTypeTag());
for (int i = 0; i < numKeys; i++) {
types.add(nestedKeyType);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
index 120edeb..b1de0b5 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
@@ -61,6 +61,8 @@
}
@Override
+ // TODO: This code has been completely rewritten in the asterix-fuzzy branch. No tests currently rely
+ // on this code, so I didn't do any cleanup here.
public JobSpecification createJobSpec(CompiledCreateIndexStatement createIndexStmt,
AqlCompiledMetadataDeclarations metadata) throws AsterixException, AlgebricksException {
JobSpecification spec = new JobSpecification();
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
index c0de096..3548877 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
@@ -99,6 +99,12 @@
// Assign op.
AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec, primaryScanOp, numNestedSecondaryKeyFields);
+ // If any of the secondary fields are nullable, then add a select op that filters nulls.
+ AlgebricksMetaOperatorDescriptor selectOp = null;
+ if (anySecondaryKeyIsNullable) {
+ selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeyFields);
+ }
+
// Create secondary RTree bulk load op.
TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec,
numNestedSecondaryKeyFields, new RTreeDataflowHelperFactory(valueProviderFactories),
@@ -107,7 +113,12 @@
// Connect the operators.
spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, secondaryBulkLoadOp, 0);
+ if (anySecondaryKeyIsNullable) {
+ spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, secondaryBulkLoadOp, 0);
+ } else {
+ spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, secondaryBulkLoadOp, 0);
+ }
spec.addRoot(secondaryBulkLoadOp);
spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
return spec;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlCompiledIndexDecl.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlCompiledIndexDecl.java
index cafd44f..c78992d 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlCompiledIndexDecl.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlCompiledIndexDecl.java
@@ -19,8 +19,11 @@
import java.util.List;
import edu.uci.ics.asterix.om.types.ARecordType;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.AUnionType;
import edu.uci.ics.asterix.om.types.IAType;
import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
public class AqlCompiledIndexDecl {
@@ -67,4 +70,17 @@
throw new AlgebricksException("Could not find field " + expr + " in the schema.");
}
+ public static Pair<IAType, Boolean> getNonNullableKeyFieldType(String expr, ARecordType recType) throws AlgebricksException {
+ IAType keyType = AqlCompiledIndexDecl.keyFieldType(expr, recType);
+ boolean nullable = false;
+ if (keyType.getTypeTag() == ATypeTag.UNION) {
+ AUnionType unionType = (AUnionType) keyType;
+ if (unionType.isNullableType()) {
+ // The non-null type is always at index 1.
+ keyType = unionType.getUnionList().get(1);
+ nullable = true;
+ }
+ }
+ return new Pair<IAType, Boolean>(keyType, nullable);
+ }
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
index dc0ffdd..5c36d71 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
@@ -387,8 +387,8 @@
}
ARecordType recType = (ARecordType) itemType;
for (i = 0; i < numSecondaryKeys; i++) {
- IAType keyType = AqlCompiledIndexDecl.keyFieldType(
- secondaryKeyFields.get(i), recType);
+ Pair<IAType, Boolean> keyTypePair = AqlCompiledIndexDecl.getNonNullableKeyFieldType(secondaryKeyFields.get(i), recType);
+ IAType keyType = keyTypePair.first;
ISerializerDeserializer keySerde = metadata.getFormat()
.getSerdeProvider().getSerializerDeserializer(keyType);
recordFields[i] = keySerde;
@@ -499,8 +499,8 @@
}
ARecordType recType = (ARecordType) itemType;
- IAType keyType = AqlCompiledIndexDecl.keyFieldType(
- secondaryKeyFields.get(0), recType);
+ Pair<IAType, Boolean> keyTypePair = AqlCompiledIndexDecl.getNonNullableKeyFieldType(secondaryKeyFields.get(0), recType);
+ IAType keyType = keyTypePair.first;
if (keyType == null) {
throw new AlgebricksException("Could not find field "
+ secondaryKeyFields.get(0) + " in the schema.");
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/CreateMBREvalFactory.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/CreateMBREvalFactory.java
index a83cf68..97930a5 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/CreateMBREvalFactory.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/CreateMBREvalFactory.java
@@ -283,9 +283,14 @@
}
}
break;
+ case NULL: {
+ out.writeByte(ATypeTag.NULL.serialize());
+ return;
+ }
default:
throw new NotImplementedException(
- "create-mbr is only implemented for POINT, LINE, POLYGON, CIRCLE and RECTANGLE.");
+ "create-mbr is only implemented for POINT, LINE, POLYGON, CIRCLE and RECTANGLE. Encountered type: "
+ + tag + ".");
}
} else {