Secondary RTree indexes can now be created on nullable fields (nulls are filtered during load). Still need to modify insert/delete jobgen to filter nulls as well.

git-svn-id: https://asterixdb.googlecode.com/svn/branches/asterix-fix-issue-9@214 eaa15691-b419-025a-1212-ee371bd00084
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceBTreeIndexSearchRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceBTreeIndexSearchRule.java
index 43aeafd..63b3b3b 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceBTreeIndexSearchRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceBTreeIndexSearchRule.java
@@ -621,7 +621,8 @@
             ARecordType itemType) throws AlgebricksException {
         List<Object> types = new ArrayList<Object>();
         for (String sk : acid.getFieldExprs()) {
-            types.add(AqlCompiledIndexDecl.keyFieldType(sk, itemType));
+            Pair<IAType, Boolean> keyPair = AqlCompiledIndexDecl.getNonNullableKeyFieldType(sk, itemType);
+            types.add(keyPair.first);
         }
         for (Triple<IEvaluatorFactory, ScalarFunctionCallExpression, IAType> t : DatasetUtils
                 .getPartitioningFunctions(ddecl)) {
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceRTreeIndexSearchRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceRTreeIndexSearchRule.java
index 6b76938..589baf4 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceRTreeIndexSearchRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceRTreeIndexSearchRule.java
@@ -178,8 +178,8 @@
 
             foundVar = findIdxExprs(adecl, primIdxFields, primIdxDecl, foundIdxExprs, outComparedVars, var, fieldName);
             if (foundVar) {
-                IAType spatialType = AqlCompiledIndexDecl.keyFieldType(fieldName, recordType);
-                dimension = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
+                Pair<IAType, Boolean> spatialTypePair = AqlCompiledIndexDecl.getNonNullableKeyFieldType(fieldName, recordType);
+                dimension = NonTaggedFormatUtil.getNumDimensions(spatialTypePair.first.getTypeTag());
             }
 
             fldPos++;
@@ -380,8 +380,8 @@
     private static List<Object> secondaryIndexTypes(AqlCompiledDatasetDecl ddecl, AqlCompiledIndexDecl acid,
             ARecordType itemType, int numKeys) throws AlgebricksException {
         List<Object> types = new ArrayList<Object>();
-        IAType keyType = AqlCompiledIndexDecl.keyFieldType(acid.getFieldExprs().get(0), itemType);
-        IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(keyType.getTypeTag());
+        Pair<IAType, Boolean> keyTypePair = AqlCompiledIndexDecl.getNonNullableKeyFieldType(acid.getFieldExprs().get(0), itemType);
+        IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(keyTypePair.first.getTypeTag());
 
         for (int i = 0; i < numKeys; i++) {
             types.add(nestedKeyType);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
index 120edeb..b1de0b5 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
@@ -61,6 +61,8 @@
     }
 
     @Override
+    // TODO: This code has been completely rewritten in the asterix-fuzzy branch. No tests currently rely
+    // on this code, so I didn't do any cleanup here.
     public JobSpecification createJobSpec(CompiledCreateIndexStatement createIndexStmt,
             AqlCompiledMetadataDeclarations metadata) throws AsterixException, AlgebricksException {
         JobSpecification spec = new JobSpecification();
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
index c0de096..3548877 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
@@ -99,6 +99,12 @@
         // Assign op.
         AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec, primaryScanOp, numNestedSecondaryKeyFields);
 
+        // If any of the secondary fields are nullable, then add a select op that filters nulls.
+        AlgebricksMetaOperatorDescriptor selectOp = null;
+        if (anySecondaryKeyIsNullable) {
+            selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeyFields);
+        }
+        
         // Create secondary RTree bulk load op.
         TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec,
                 numNestedSecondaryKeyFields, new RTreeDataflowHelperFactory(valueProviderFactories),
@@ -107,7 +113,12 @@
         // Connect the operators.
         spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
         spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
-        spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, secondaryBulkLoadOp, 0);
+        if (anySecondaryKeyIsNullable) {
+            spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
+            spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, secondaryBulkLoadOp, 0);
+        } else {
+            spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, secondaryBulkLoadOp, 0);
+        }
         spec.addRoot(secondaryBulkLoadOp);
         spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
         return spec;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlCompiledIndexDecl.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlCompiledIndexDecl.java
index cafd44f..c78992d 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlCompiledIndexDecl.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlCompiledIndexDecl.java
@@ -19,8 +19,11 @@
 import java.util.List;
 
 import edu.uci.ics.asterix.om.types.ARecordType;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.AUnionType;
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
 
 public class AqlCompiledIndexDecl {
 
@@ -67,4 +70,17 @@
         throw new AlgebricksException("Could not find field " + expr + " in the schema.");
     }
 
+    public static Pair<IAType, Boolean> getNonNullableKeyFieldType(String expr, ARecordType recType) throws AlgebricksException {
+        IAType keyType = AqlCompiledIndexDecl.keyFieldType(expr, recType);
+        boolean nullable = false;
+        if (keyType.getTypeTag() == ATypeTag.UNION) {
+            AUnionType unionType = (AUnionType) keyType;
+            if (unionType.isNullableType()) {
+                // The non-null type is always at index 1.
+                keyType = unionType.getUnionList().get(1);
+                nullable = true;
+            }
+        }
+        return new Pair<IAType, Boolean>(keyType, nullable);
+    }
 }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
index dc0ffdd..5c36d71 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
@@ -387,8 +387,8 @@
 			}
 			ARecordType recType = (ARecordType) itemType;
 			for (i = 0; i < numSecondaryKeys; i++) {
-				IAType keyType = AqlCompiledIndexDecl.keyFieldType(
-						secondaryKeyFields.get(i), recType);
+				Pair<IAType, Boolean> keyTypePair = AqlCompiledIndexDecl.getNonNullableKeyFieldType(secondaryKeyFields.get(i), recType);
+				IAType keyType = keyTypePair.first;
 				ISerializerDeserializer keySerde = metadata.getFormat()
 						.getSerdeProvider().getSerializerDeserializer(keyType);
 				recordFields[i] = keySerde;
@@ -499,8 +499,8 @@
 			}
 			ARecordType recType = (ARecordType) itemType;
 
-			IAType keyType = AqlCompiledIndexDecl.keyFieldType(
-					secondaryKeyFields.get(0), recType);
+			Pair<IAType, Boolean> keyTypePair = AqlCompiledIndexDecl.getNonNullableKeyFieldType(secondaryKeyFields.get(0), recType);
+			IAType keyType = keyTypePair.first;
 			if (keyType == null) {
 				throw new AlgebricksException("Could not find field "
 						+ secondaryKeyFields.get(0) + " in the schema.");
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/CreateMBREvalFactory.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/CreateMBREvalFactory.java
index a83cf68..97930a5 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/CreateMBREvalFactory.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/CreateMBREvalFactory.java
@@ -283,9 +283,14 @@
                                     }
                                 }
                                 break;
+                            case NULL: {
+                            	out.writeByte(ATypeTag.NULL.serialize());
+                            	return;
+                            }
                             default:
                                 throw new NotImplementedException(
-                                        "create-mbr is only implemented for POINT, LINE, POLYGON, CIRCLE and RECTANGLE.");
+									"create-mbr is only implemented for POINT, LINE, POLYGON, CIRCLE and RECTANGLE. Encountered type: "
+											+ tag + ".");
 
                         }
                     } else {