[ASTERIXDB-2791][IDX] Add option to exclude unknowns in secondary indexes
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
- add EXCLUDE UNKNOWN KEY and INCLUDE UNKNOWN KEY to CREATE INDEX statement
to allow user to exclude/include unknonwn keys.
- bring back the filter when creating the job spec for BTrees.
Use the filter when unknown keys should be excluded.
- For UPSERT, pass the filter for previous tuple, as well, to determine
if the previous tuple's index entry should be processed.
Change-Id: Id40de17d1392510f6ebfea5bd81037a9305895af
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12624
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Reviewed-by: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index f8a6cd8..7567bbd 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -313,6 +313,7 @@
// Set our key variables and expressions for non-array indexes. Our secondary keys for array indexes will
// always be an empty list.
List<LogicalVariable> secondaryKeyVars = new ArrayList<>();
+ List<LogicalVariable> beforeOpSecondaryKeyVars = new ArrayList<>();
List<Mutable<ILogicalExpression>> secondaryExpressions = new ArrayList<>();
List<Mutable<ILogicalExpression>> beforeOpSecondaryExpressions = new ArrayList<>();
ILogicalOperator replicateOutput;
@@ -324,23 +325,30 @@
secondaryKeyVars.add(skVar);
VariableReferenceExpression skVarRef = new VariableReferenceExpression(skVar);
skVarRef.setSourceLocation(sourceLoc);
- secondaryExpressions.add(new MutableObject<ILogicalExpression>(skVarRef));
+ secondaryExpressions.add(new MutableObject<>(skVarRef));
if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) {
- VariableReferenceExpression varRef =
- new VariableReferenceExpression(fieldVarsForBeforeOperation.get(indexFieldId));
+ LogicalVariable beforeKeyVar = fieldVarsForBeforeOperation.get(indexFieldId);
+ beforeOpSecondaryKeyVars.add(beforeKeyVar);
+ VariableReferenceExpression varRef = new VariableReferenceExpression(beforeKeyVar);
varRef.setSourceLocation(sourceLoc);
- beforeOpSecondaryExpressions.add(new MutableObject<ILogicalExpression>(varRef));
+ beforeOpSecondaryExpressions.add(new MutableObject<>(varRef));
}
}
}
IndexInsertDeleteUpsertOperator indexUpdate;
if (index.getIndexType() != IndexType.RTREE) {
+ // B-Tree, inverted index, array index
// Create an expression per key
- Mutable<ILogicalExpression> filterExpression = (primaryIndexModificationOp.getOperation() == Kind.UPSERT
- || index.getIndexType() == IndexType.BTREE) ? null
- : createFilterExpression(secondaryKeyVars, context.getOutputTypeEnvironment(currentTop),
- index.getIndexDetails().isOverridingKeyFieldTypes());
+ Mutable<ILogicalExpression> filterExpression =
+ createFilterExpression(index, secondaryKeyVars, context.getOutputTypeEnvironment(currentTop),
+ index.getIndexDetails().isOverridingKeyFieldTypes());
+ Mutable<ILogicalExpression> beforeOpFilterExpression = null;
+ if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) {
+ beforeOpFilterExpression = createFilterExpression(index, beforeOpSecondaryKeyVars,
+ context.getOutputTypeEnvironment(currentTop),
+ index.getIndexDetails().isOverridingKeyFieldTypes());
+ }
DataSourceIndex dataSourceIndex = new DataSourceIndex(index, dataverseName, datasetName, mp);
// Introduce the TokenizeOperator only when doing bulk-load,
@@ -406,8 +414,8 @@
indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex,
OperatorManipulationUtil
.cloneExpressions(primaryIndexModificationOp.getPrimaryKeyExpressions()),
- tokenizeKeyExprs, filterExpression, primaryIndexModificationOp.getOperation(),
- primaryIndexModificationOp.isBulkload(),
+ tokenizeKeyExprs, filterExpression, beforeOpFilterExpression,
+ primaryIndexModificationOp.getOperation(), primaryIndexModificationOp.isBulkload(),
primaryIndexModificationOp.getAdditionalNonFilteringExpressions() == null ? 0
: primaryIndexModificationOp.getAdditionalNonFilteringExpressions().size());
indexUpdate.setSourceLocation(sourceLoc);
@@ -419,8 +427,8 @@
indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex,
OperatorManipulationUtil
.cloneExpressions(primaryIndexModificationOp.getPrimaryKeyExpressions()),
- secondaryExpressions, filterExpression, primaryIndexModificationOp.getOperation(),
- primaryIndexModificationOp.isBulkload(),
+ secondaryExpressions, filterExpression, beforeOpFilterExpression,
+ primaryIndexModificationOp.getOperation(), primaryIndexModificationOp.isBulkload(),
primaryIndexModificationOp.getAdditionalNonFilteringExpressions() == null ? 0
: primaryIndexModificationOp.getAdditionalNonFilteringExpressions().size());
indexUpdate.setSourceLocation(sourceLoc);
@@ -453,7 +461,7 @@
// If there exists a filter expression, add it to the top of our nested plan.
filterExpression = (primaryIndexModificationOp.getOperation() == Kind.UPSERT) ? null
- : createFilterExpression(unnestSIDXBranch.lastFieldVars,
+ : createAnyUnknownFilterExpression(unnestSIDXBranch.lastFieldVars,
context.getOutputTypeEnvironment(unnestSIDXBranch.currentTop),
index.getIndexDetails().isOverridingKeyFieldTypes());
if (filterExpression != null) {
@@ -498,7 +506,7 @@
indexUpdate.setSecondaryKeyExprs(secondaryExpressions);
// Update the filter expression to include these new keys.
- filterExpression = createFilterExpression(unnestSIDXBranch.lastFieldVars,
+ filterExpression = createAnyUnknownFilterExpression(unnestSIDXBranch.lastFieldVars,
context.getOutputTypeEnvironment(unnestSIDXBranch.currentTop),
index.getIndexDetails().isOverridingKeyFieldTypes());
indexUpdate.setFilterExpression(filterExpression);
@@ -560,8 +568,11 @@
assignCoordinates.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
context.computeAndSetTypeEnvironmentForOperator(assignCoordinates);
replicateOutput = assignCoordinates;
- Mutable<ILogicalExpression> filterExpression = null;
+ boolean forceFilter = keyPairType.second;
+ Mutable<ILogicalExpression> filterExpression = createAnyUnknownFilterExpression(keyVarList,
+ context.getOutputTypeEnvironment(assignCoordinates), forceFilter);
AssignOperator originalAssignCoordinates = null;
+ Mutable<ILogicalExpression> beforeOpFilterExpression = null;
// We do something similar for beforeOp key if the operation is an upsert
if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) {
List<LogicalVariable> originalKeyVarList = new ArrayList<>();
@@ -592,19 +603,15 @@
originalAssignCoordinates.setSourceLocation(sourceLoc);
originalAssignCoordinates.getInputs().add(new MutableObject<ILogicalOperator>(assignCoordinates));
context.computeAndSetTypeEnvironmentForOperator(originalAssignCoordinates);
- } else {
- // We must enforce the filter if the originating spatial type is
- // nullable.
- boolean forceFilter = keyPairType.second;
- filterExpression = createFilterExpression(keyVarList,
- context.getOutputTypeEnvironment(assignCoordinates), forceFilter);
+ beforeOpFilterExpression = createAnyUnknownFilterExpression(originalKeyVarList,
+ context.getOutputTypeEnvironment(originalAssignCoordinates), forceFilter);
}
DataSourceIndex dataSourceIndex = new DataSourceIndex(index, dataverseName, datasetName, mp);
indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex,
OperatorManipulationUtil
.cloneExpressions(primaryIndexModificationOp.getPrimaryKeyExpressions()),
- secondaryExpressions, filterExpression, primaryIndexModificationOp.getOperation(),
- primaryIndexModificationOp.isBulkload(),
+ secondaryExpressions, filterExpression, beforeOpFilterExpression,
+ primaryIndexModificationOp.getOperation(), primaryIndexModificationOp.isBulkload(),
primaryIndexModificationOp.getAdditionalNonFilteringExpressions() == null ? 0
: primaryIndexModificationOp.getAdditionalNonFilteringExpressions().size());
indexUpdate.setSourceLocation(sourceLoc);
@@ -937,11 +944,42 @@
return new MutableObject<>(new ConstantExpression(new AsterixConstantValue(constantObject)));
}
- private Mutable<ILogicalExpression> createFilterExpression(List<LogicalVariable> secondaryKeyVars,
+ private Mutable<ILogicalExpression> createFilterExpression(Index index, List<LogicalVariable> secondaryKeyVars,
IVariableTypeEnvironment typeEnv, boolean forceFilter) throws AlgebricksException {
+ IndexType indexType = index.getIndexType();
+ if (indexType == IndexType.BTREE) {
+ if (index.isPrimaryKeyIndex()) {
+ return createAnyUnknownFilterExpression(secondaryKeyVars, typeEnv, forceFilter);
+ } else {
+ Boolean excludeUnknownKey = ((Index.ValueIndexDetails) index.getIndexDetails()).isExcludeUnknownKey();
+ boolean excludeUnknown = excludeUnknownKey != null && excludeUnknownKey;
+ return createAllUnknownFilterExpression(secondaryKeyVars, typeEnv, forceFilter, excludeUnknown);
+ }
+ } else {
+ // inverted index && array index
+ return createAnyUnknownFilterExpression(secondaryKeyVars, typeEnv, forceFilter);
+ }
+ }
+
+ private Mutable<ILogicalExpression> createAnyUnknownFilterExpression(List<LogicalVariable> secondaryKeyVars,
+ IVariableTypeEnvironment typeEnv, boolean forceFilter) throws AlgebricksException {
+ return createFilterExpression(secondaryKeyVars, typeEnv, forceFilter, true, BuiltinFunctions.AND);
+ }
+
+ private Mutable<ILogicalExpression> createAllUnknownFilterExpression(List<LogicalVariable> secondaryKeyVars,
+ IVariableTypeEnvironment typeEnv, boolean forceFilter, boolean excludeUnknownKey)
+ throws AlgebricksException {
+ return createFilterExpression(secondaryKeyVars, typeEnv, forceFilter, excludeUnknownKey, BuiltinFunctions.OR);
+ }
+
+ private Mutable<ILogicalExpression> createFilterExpression(List<LogicalVariable> secondaryKeyVars,
+ IVariableTypeEnvironment typeEnv, boolean forceFilter, boolean excludeUnknownKey,
+ FunctionIdentifier combiner) throws AlgebricksException {
+ if (!excludeUnknownKey) {
+ return null;
+ }
List<Mutable<ILogicalExpression>> filterExpressions = new ArrayList<>();
- // Add 'is not null' to all nullable secondary index keys as a filtering
- // condition.
+ // Add 'is not null' to all nullable secondary index keys as a filtering condition
for (LogicalVariable secondaryKeyVar : secondaryKeyVars) {
IAType secondaryKeyType = (IAType) typeEnv.getVarType(secondaryKeyVar);
if (!NonTaggedFormatUtil.isOptional(secondaryKeyType) && !forceFilter) {
@@ -965,11 +1003,11 @@
}
Mutable<ILogicalExpression> filterExpression;
if (filterExpressions.size() > 1) {
- // Create a conjunctive condition.
- ScalarFunctionCallExpression andExpr = new ScalarFunctionCallExpression(
- FunctionUtil.getFunctionInfo(BuiltinFunctions.AND), filterExpressions);
- andExpr.setSourceLocation(sourceLoc);
- filterExpression = new MutableObject<>(andExpr);
+ // Combine the conditions
+ ScalarFunctionCallExpression combinerExpr = new ScalarFunctionCallExpression(
+ BuiltinFunctions.getBuiltinFunctionInfo(combiner), filterExpressions);
+ combinerExpr.setSourceLocation(sourceLoc);
+ filterExpression = new MutableObject<>(combinerExpr);
} else {
filterExpression = filterExpressions.get(0);
}
diff --git a/asterixdb/asterix-app/data/misc/1.adm b/asterixdb/asterix-app/data/misc/1.adm
new file mode 100644
index 0000000..93c166a
--- /dev/null
+++ b/asterixdb/asterix-app/data/misc/1.adm
@@ -0,0 +1,9 @@
+{"id": 1, "int_f": 1 , "str_f": "1"}
+{"id": 2, "int_f": null, "str_f": "2"}
+{"id": 3 , "str_f": "3"}
+{"id": 4, "int_f": 4 , "str_f": null}
+{"id": 5, "int_f": 5 }
+{"id": 6, "int_f": null }
+{"id": 7 , "str_f": null}
+{"id": 8, "int_f": null, "str_f": null}
+{"id": 9 ,"other_f": "other" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index e79a4fd..173b6d4 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -1234,6 +1234,11 @@
indexFieldTypes.add(fieldTypes);
}
+ boolean unknownKeyOptionAllowed = indexType == IndexType.BTREE && !isSecondaryPrimary;
+ if (stmtCreateIndex.hasExcludeUnknownKey() && !unknownKeyOptionAllowed) {
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
+ "can only specify exclude/include unknown key for B-Tree indexes");
+ }
Index.IIndexDetails indexDetails;
if (Index.IndexCategory.of(indexType) == Index.IndexCategory.ARRAY) {
if (!hadUnnest) {
@@ -1280,7 +1285,7 @@
switch (Index.IndexCategory.of(indexType)) {
case VALUE:
indexDetails = new Index.ValueIndexDetails(keyFieldNames, keyFieldSourceIndicators,
- keyFieldTypes, overridesFieldTypes);
+ keyFieldTypes, overridesFieldTypes, stmtCreateIndex.isExcludeUnknownKey());
break;
case TEXT:
indexDetails = new Index.TextIndexDetails(keyFieldNames, keyFieldSourceIndicators,
@@ -1488,10 +1493,15 @@
// Get snapshot from External File System
externalFilesSnapshot = ExternalIndexingOperations.getSnapshotFromExternalFileSystem(ds);
// Add an entry for the files index
+ Index.IndexCategory indexCategory = Index.IndexCategory.of(index.getIndexType());
+ Boolean excludeUnknownKey = null;
+ if (indexCategory == Index.IndexCategory.VALUE) {
+ excludeUnknownKey = ((Index.ValueIndexDetails) index.getIndexDetails()).isExcludeUnknownKey();
+ }
filesIndex = new Index(index.getDataverseName(), index.getDatasetName(),
IndexingConstants.getFilesIndexName(index.getDatasetName()), IndexType.BTREE,
new Index.ValueIndexDetails(ExternalIndexingOperations.FILE_INDEX_FIELD_NAMES, null,
- ExternalIndexingOperations.FILE_INDEX_FIELD_TYPES, false),
+ ExternalIndexingOperations.FILE_INDEX_FIELD_TYPES, false, excludeUnknownKey),
false, false, MetadataUtil.PENDING_ADD_OP);
MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), filesIndex);
// Add files to the external files index
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
index c668fb6..d4e9cf8 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
@@ -28,7 +28,6 @@
import org.apache.asterix.app.nc.NCAppRuntimeContext;
import org.apache.asterix.app.nc.TransactionSubsystem;
-import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.config.TransactionProperties;
import org.apache.asterix.common.context.DatasetLifecycleManager;
import org.apache.asterix.common.context.IStorageComponentProvider;
@@ -718,10 +717,8 @@
indicator == Index.RECORD_INDICATOR ? recordType.getFieldNames() : metaType.getFieldNames();
keyFieldNames.add(Arrays.asList(fieldNames[primaryKeyIndexes[i]]));
}
- index = new Index(dataset.getDataverseName(), dataset.getDatasetName(), dataset.getDatasetName(),
- IndexType.BTREE,
- new Index.ValueIndexDetails(keyFieldNames, primaryKeyIndicators, keyFieldTypes, false), false, true,
- MetadataUtil.PENDING_NO_OP);
+ index = Index.createPrimaryIndex(dataset.getDataverseName(), dataset.getDatasetName(), keyFieldNames,
+ primaryKeyIndicators, keyFieldTypes, MetadataUtil.PENDING_NO_OP);
List<String> nodes = Collections.singletonList(ExecutionTestUtil.integrationUtil.ncs[0].getId());
CcApplicationContext appCtx =
(CcApplicationContext) ExecutionTestUtil.integrationUtil.cc.getApplicationContext();
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/CheckpointInSecondaryIndexTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/CheckpointInSecondaryIndexTest.java
index 1913233..caab0b8 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/CheckpointInSecondaryIndexTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/CheckpointInSecondaryIndexTest.java
@@ -171,7 +171,7 @@
partitioningKeys, null, null, null, false, null, null),
null, DatasetType.INTERNAL, DATASET_ID, 0);
secondaryIndex = new Index(dvName, DATASET_NAME, INDEX_NAME, INDEX_TYPE, INDEX_FIELD_NAMES,
- INDEX_FIELD_INDICATORS, INDEX_FIELD_TYPES, false, false, false, 0);
+ INDEX_FIELD_INDICATORS, INDEX_FIELD_TYPES, false, false, false, 0, false);
taskCtx = null;
primaryIndexDataflowHelper = null;
secondaryIndexDataflowHelper = null;
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ConcurrentInsertTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ConcurrentInsertTest.java
index 76dd035..bea1791 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ConcurrentInsertTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ConcurrentInsertTest.java
@@ -128,7 +128,8 @@
Index secondaryIndexEntity = new Index(StorageTestUtils.DATASET.getDataverseName(),
StorageTestUtils.DATASET.getDatasetName(), "TestIndex", IndexType.BTREE,
Arrays.asList(Arrays.asList(StorageTestUtils.RECORD_TYPE.getFieldNames()[1])),
- Arrays.asList(Index.RECORD_INDICATOR), Arrays.asList(BuiltinType.AINT64), false, false, false, 0);
+ Arrays.asList(Index.RECORD_INDICATOR), Arrays.asList(BuiltinType.AINT64), false, false, false, 0,
+ false);
SecondaryIndexInfo secondaryIndexInfo =
nc.createSecondaryIndex(primaryIndexInfo, secondaryIndexEntity, StorageTestUtils.STORAGE_MANAGER, 0);
@@ -142,7 +143,7 @@
Index primaryKeyIndexEntity = new Index(StorageTestUtils.DATASET.getDataverseName(),
StorageTestUtils.DATASET.getDatasetName(), "PrimaryKeyIndex", IndexType.BTREE, Arrays.asList(),
- Arrays.asList(), Arrays.asList(), false, false, false, 0);
+ Arrays.asList(), Arrays.asList(), false, false, false, 0, null);
SecondaryIndexInfo primaryKeyIndexInfo =
nc.createSecondaryIndex(primaryIndexInfo, primaryKeyIndexEntity, StorageTestUtils.STORAGE_MANAGER, 0);
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
index 2aad416..f408758 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
@@ -187,7 +187,7 @@
dataset = StorageTestUtils.DATASET;
secondaryIndexEntity = new Index(dataset.getDataverseName(), dataset.getDatasetName(), SECONDARY_INDEX_NAME,
SECONDARY_INDEX_TYPE, SECONDARY_INDEX_FIELD_NAMES, SECONDARY_INDEX_FIELD_INDICATORS,
- SECONDARY_INDEX_FIELD_TYPES, false, false, false, 0);
+ SECONDARY_INDEX_FIELD_TYPES, false, false, false, 0, false);
primaryIndexInfos = new PrimaryIndexInfo[NUM_PARTITIONS];
secondaryIndexInfo = new SecondaryIndexInfo[NUM_PARTITIONS];
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/MultiPartitionLSMIndexTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/MultiPartitionLSMIndexTest.java
index ef3a7cb..42ddd95 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/MultiPartitionLSMIndexTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/MultiPartitionLSMIndexTest.java
@@ -161,7 +161,7 @@
partitioningKeys, null, null, null, false, null, null),
null, DatasetType.INTERNAL, DATASET_ID, 0);
secondaryIndex = new Index(dvName, DATASET_NAME, INDEX_NAME, INDEX_TYPE, INDEX_FIELD_NAMES,
- INDEX_FIELD_INDICATORS, INDEX_FIELD_TYPES, false, false, false, 0);
+ INDEX_FIELD_INDICATORS, INDEX_FIELD_TYPES, false, false, false, 0, false);
taskCtxs = new IHyracksTaskContext[NUM_PARTITIONS];
primaryIndexDataflowHelpers = new IIndexDataflowHelper[NUM_PARTITIONS];
secondaryIndexDataflowHelpers = new IIndexDataflowHelper[NUM_PARTITIONS];
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta05/meta05.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta05/meta05.1.adm
index f770710..5f1d071 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta05/meta05.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta05/meta05.1.adm
@@ -1,2 +1,2 @@
-{ "DataverseName": "testdv", "DatasetName": "t1", "IndexName": "idx1", "IndexStructure": "BTREE", "SearchKey": [ [ "name" ] ], "IsPrimary": false, "Timestamp": "Tue Jun 21 15:54:27 PDT 2016", "PendingOp": 0 }
+{ "DataverseName": "testdv", "DatasetName": "t1", "IndexName": "idx1", "IndexStructure": "BTREE", "SearchKey": [ [ "name" ] ], "IsPrimary": false, "Timestamp": "Tue Jun 21 15:54:27 PDT 2016", "PendingOp": 0, "ExcludeUnknownKey": false }
{ "DataverseName": "testdv", "DatasetName": "t1", "IndexName": "t1", "IndexStructure": "BTREE", "SearchKey": [ [ "id" ] ], "IsPrimary": true, "Timestamp": "Tue Jun 21 15:54:27 PDT 2016", "PendingOp": 0 }
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta16/meta16.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta16/meta16.1.adm
index dab91a3..300715d1 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta16/meta16.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta16/meta16.1.adm
@@ -1,2 +1,2 @@
-{ "DataverseName": "testdv", "DatasetName": "t1", "IndexName": "idx1", "IndexStructure": "BTREE", "SearchKey": [ [ "name" ] ], "IsPrimary": false, "Timestamp": "Tue Jun 21 15:54:28 PDT 2016", "PendingOp": 0, "SearchKeyType": [ "string" ], "IsEnforced": true }
+{ "DataverseName": "testdv", "DatasetName": "t1", "IndexName": "idx1", "IndexStructure": "BTREE", "SearchKey": [ [ "name" ] ], "IsPrimary": false, "Timestamp": "Tue Jun 21 15:54:28 PDT 2016", "PendingOp": 0, "SearchKeyType": [ "string" ], "IsEnforced": true, "ExcludeUnknownKey": false }
{ "DataverseName": "testdv", "DatasetName": "t1", "IndexName": "t1", "IndexStructure": "BTREE", "SearchKey": [ [ "id" ] ], "IsPrimary": true, "Timestamp": "Tue Jun 21 15:54:28 PDT 2016", "PendingOp": 0 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.01.ddl.sqlpp
new file mode 100644
index 0000000..89abc08
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.01.ddl.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// test that secondary indexes can include or exclude entries that have unknown keys
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE DATASET ds1(id int not unknown) OPEN TYPE PRIMARY KEY id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.02.update.sqlpp
new file mode 100644
index 0000000..0ac8895
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.02.update.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+INSERT INTO ds1 ([
+{"id": 1, "int_f": 1 , "str_f": "1"},
+{"id": 2, "int_f": null, "str_f": "2"},
+{"id": 3 , "str_f": "3"},
+{"id": 4, "int_f": 4 , "str_f": null},
+{"id": 5, "int_f": 5 },
+{"id": 6, "int_f": null },
+{"id": 7 , "str_f": null},
+{"id": 8, "int_f": null, "str_f": null},
+{"id": 9 ,"other_f": "other" }
+]);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.03.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.03.ddl.sqlpp
new file mode 100644
index 0000000..44546cf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.03.ddl.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+CREATE INDEX exclude_unknown_idx1 ON ds1(int_f: int) exclude unknown key;
+CREATE INDEX exclude_unknown_idx2 ON ds1(int_f: int, str_f: string) exclude unknown key;
+
+CREATE INDEX include_unknown_idx1 ON ds1(int_f: int) include unknown key;
+CREATE INDEX include_unknown_idx2 ON ds1(int_f: int, str_f: string) include unknown key;
+
+CREATE INDEX idx1 ON ds1(int_f: int);
+CREATE INDEX idx2 ON ds1(int_f: int, str_f: string);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.04.query.sqlpp
new file mode 100644
index 0000000..02c8708
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.04.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "exclude_unknown_idx1") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.05.query.sqlpp
new file mode 100644
index 0000000..826c8a6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.05.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "exclude_unknown_idx2") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.06.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.06.query.sqlpp
new file mode 100644
index 0000000..b3ed16f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.06.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "include_unknown_idx1") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.07.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.07.query.sqlpp
new file mode 100644
index 0000000..91543ab
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.07.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "include_unknown_idx2") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.08.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.08.query.sqlpp
new file mode 100644
index 0000000..14f2ebd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.08.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "idx1") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.09.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.09.query.sqlpp
new file mode 100644
index 0000000..845ee14
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.09.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "idx2") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.10.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.10.query.sqlpp
new file mode 100644
index 0000000..edc8ece
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.10.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+SELECT COUNT(*) AS cnt FROM ds1;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.11.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.11.query.sqlpp
new file mode 100644
index 0000000..8f4c38b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.11.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+SELECT i.DataverseName, i.IndexName, i.ExcludeUnknownKey FROM Metadata.`Index` i WHERE i.DataverseName = "test" ORDER BY i.IndexName;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.99.ddl.sqlpp
new file mode 100644
index 0000000..36b2bab
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/create-index-unknown-key/create-index-unknown-key.99.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.01.ddl.sqlpp
new file mode 100644
index 0000000..9d3f4bf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.01.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// test that secondary indexes can include or exclude entries that have unknown keys
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE DATASET ds1(id int not unknown) OPEN TYPE PRIMARY KEY id;
+
+CREATE INDEX exclude_unknown_idx1 ON ds1(int_f: int) exclude unknown key;
+CREATE INDEX exclude_unknown_idx2 ON ds1(int_f: int, str_f: string) exclude unknown key;
+
+CREATE INDEX include_unknown_idx1 ON ds1(int_f: int) include unknown key;
+CREATE INDEX include_unknown_idx2 ON ds1(int_f: int, str_f: string) include unknown key;
+
+CREATE INDEX idx1 ON ds1(int_f: int);
+CREATE INDEX idx2 ON ds1(int_f: int, str_f: string);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.02.update.sqlpp
new file mode 100644
index 0000000..0ac8895
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.02.update.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+INSERT INTO ds1 ([
+{"id": 1, "int_f": 1 , "str_f": "1"},
+{"id": 2, "int_f": null, "str_f": "2"},
+{"id": 3 , "str_f": "3"},
+{"id": 4, "int_f": 4 , "str_f": null},
+{"id": 5, "int_f": 5 },
+{"id": 6, "int_f": null },
+{"id": 7 , "str_f": null},
+{"id": 8, "int_f": null, "str_f": null},
+{"id": 9 ,"other_f": "other" }
+]);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.04.query.sqlpp
new file mode 100644
index 0000000..02c8708
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.04.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "exclude_unknown_idx1") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.05.query.sqlpp
new file mode 100644
index 0000000..826c8a6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.05.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "exclude_unknown_idx2") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.06.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.06.query.sqlpp
new file mode 100644
index 0000000..b3ed16f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.06.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "include_unknown_idx1") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.07.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.07.query.sqlpp
new file mode 100644
index 0000000..91543ab
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.07.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "include_unknown_idx2") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.08.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.08.query.sqlpp
new file mode 100644
index 0000000..14f2ebd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.08.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "idx1") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.09.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.09.query.sqlpp
new file mode 100644
index 0000000..845ee14
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.09.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "idx2") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.10.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.10.query.sqlpp
new file mode 100644
index 0000000..edc8ece
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.10.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+SELECT COUNT(*) AS cnt FROM ds1;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.11.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.11.query.sqlpp
new file mode 100644
index 0000000..8f4c38b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.11.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+SELECT i.DataverseName, i.IndexName, i.ExcludeUnknownKey FROM Metadata.`Index` i WHERE i.DataverseName = "test" ORDER BY i.IndexName;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.99.ddl.sqlpp
new file mode 100644
index 0000000..36b2bab
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/insert-into-index-unknown-key/insert-into-index-unknown-key.99.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.01.ddl.sqlpp
new file mode 100644
index 0000000..9d3f4bf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.01.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// test that secondary indexes can include or exclude entries that have unknown keys
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE DATASET ds1(id int not unknown) OPEN TYPE PRIMARY KEY id;
+
+CREATE INDEX exclude_unknown_idx1 ON ds1(int_f: int) exclude unknown key;
+CREATE INDEX exclude_unknown_idx2 ON ds1(int_f: int, str_f: string) exclude unknown key;
+
+CREATE INDEX include_unknown_idx1 ON ds1(int_f: int) include unknown key;
+CREATE INDEX include_unknown_idx2 ON ds1(int_f: int, str_f: string) include unknown key;
+
+CREATE INDEX idx1 ON ds1(int_f: int);
+CREATE INDEX idx2 ON ds1(int_f: int, str_f: string);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.02.update.sqlpp
new file mode 100644
index 0000000..12aaddb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.02.update.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+LOAD DATASET ds1 USING localfs((`path`=`asterix_nc1://data/misc/1.adm`),(`format`=`adm`));
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.04.query.sqlpp
new file mode 100644
index 0000000..02c8708
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.04.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "exclude_unknown_idx1") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.05.query.sqlpp
new file mode 100644
index 0000000..826c8a6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.05.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "exclude_unknown_idx2") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.06.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.06.query.sqlpp
new file mode 100644
index 0000000..b3ed16f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.06.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "include_unknown_idx1") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.07.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.07.query.sqlpp
new file mode 100644
index 0000000..91543ab
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.07.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "include_unknown_idx2") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.08.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.08.query.sqlpp
new file mode 100644
index 0000000..14f2ebd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.08.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "idx1") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.09.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.09.query.sqlpp
new file mode 100644
index 0000000..845ee14
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.09.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "idx2") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.10.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.10.query.sqlpp
new file mode 100644
index 0000000..edc8ece
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.10.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+SELECT COUNT(*) AS cnt FROM ds1;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.11.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.11.query.sqlpp
new file mode 100644
index 0000000..8f4c38b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.11.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+SELECT i.DataverseName, i.IndexName, i.ExcludeUnknownKey FROM Metadata.`Index` i WHERE i.DataverseName = "test" ORDER BY i.IndexName;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.99.ddl.sqlpp
new file mode 100644
index 0000000..36b2bab
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/load-into-index-unknown-key/load-into-index-unknown-key.99.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.01.ddl.sqlpp
new file mode 100644
index 0000000..fd4edb5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.01.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// test that secondary indexes can include or exclude entries that have unknown keys
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE DATASET ds1(id int not unknown) OPEN TYPE PRIMARY KEY id;
+
+CREATE INDEX exclude_unknown_idx1 ON ds1(int_f: int) EXCLUDE UNKNOWN KEY;
+CREATE INDEX exclude_unknown_idx2 ON ds1(int_f: int, str_f: string) EXCLUDE UNKNOWN KEY;
+
+CREATE INDEX include_unknown_idx1 ON ds1(int_f: int) INCLUDE UNKNOWN KEY;
+CREATE INDEX include_unknown_idx2 ON ds1(int_f: int, str_f: string) INCLUDE UNKNOWN KEY;
+
+CREATE INDEX idx1 ON ds1(int_f: int);
+CREATE INDEX idx2 ON ds1(int_f: int, str_f: string);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.02.update.sqlpp
new file mode 100644
index 0000000..ddd4565
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.02.update.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+UPSERT INTO ds1 ([
+{"id": 1, "int_f": 1 , "str_f": "1"},
+{"id": 2, "int_f": null, "str_f": "2"},
+{"id": 3 , "str_f": "3"},
+{"id": 4, "int_f": 4 , "str_f": null},
+{"id": 5, "int_f": 5 },
+{"id": 6, "int_f": null },
+{"id": 7 , "str_f": null},
+{"id": 8, "int_f": null, "str_f": null},
+{"id": 9 ,"other_f": "other" }
+]);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.04.query.sqlpp
new file mode 100644
index 0000000..02c8708
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.04.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "exclude_unknown_idx1") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.05.query.sqlpp
new file mode 100644
index 0000000..826c8a6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.05.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "exclude_unknown_idx2") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.06.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.06.query.sqlpp
new file mode 100644
index 0000000..b3ed16f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.06.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "include_unknown_idx1") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.07.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.07.query.sqlpp
new file mode 100644
index 0000000..91543ab
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.07.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "include_unknown_idx2") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.08.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.08.query.sqlpp
new file mode 100644
index 0000000..14f2ebd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.08.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "idx1") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.09.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.09.query.sqlpp
new file mode 100644
index 0000000..845ee14
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.09.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "idx2") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.10.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.10.query.sqlpp
new file mode 100644
index 0000000..edc8ece
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.10.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+SELECT COUNT(*) AS cnt FROM ds1;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.11.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.11.query.sqlpp
new file mode 100644
index 0000000..8f4c38b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.11.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+SELECT i.DataverseName, i.IndexName, i.ExcludeUnknownKey FROM Metadata.`Index` i WHERE i.DataverseName = "test" ORDER BY i.IndexName;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.99.ddl.sqlpp
new file mode 100644
index 0000000..36b2bab
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/upsert-into-index-unknown-key/upsert-into-index-unknown-key.99.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create_dataset_with_filter_on_meta/create_dataset_with_filter_on_meta.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create_dataset_with_filter_on_meta/create_dataset_with_filter_on_meta.1.adm
index fcd415b..e8c9dfa 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create_dataset_with_filter_on_meta/create_dataset_with_filter_on_meta.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create_dataset_with_filter_on_meta/create_dataset_with_filter_on_meta.1.adm
@@ -1,3 +1,3 @@
{ "DataverseName": "KeyVerse", "DatasetName": "KVStore", "IndexName": "KVStore", "IndexStructure": "BTREE", "SearchKey": [ [ "key" ] ], "IsPrimary": true, "Timestamp": "Mon Sep 07 18:14:11 PDT 2020", "PendingOp": 0, "SearchKeySourceIndicator": [ 1 ] }
-{ "DataverseName": "KeyVerse", "DatasetName": "KVStore", "IndexName": "s_location", "IndexStructure": "RTREE", "SearchKey": [ [ "location" ] ], "IsPrimary": false, "Timestamp": "Mon Sep 07 18:14:11 PDT 2020", "PendingOp": 0 }
-{ "DataverseName": "KeyVerse", "DatasetName": "KVStore", "IndexName": "s_rating", "IndexStructure": "BTREE", "SearchKey": [ [ "area_code" ] ], "IsPrimary": false, "Timestamp": "Mon Sep 07 18:14:11 PDT 2020", "PendingOp": 0 }
+{ "DataverseName": "KeyVerse", "DatasetName": "KVStore", "IndexName": "s_location", "IndexStructure": "RTREE", "SearchKey": [ [ "location" ] ], "IsPrimary": false, "Timestamp": "Tue Aug 03 21:39:47 AST 2021", "PendingOp": 0 }
+{ "DataverseName": "KeyVerse", "DatasetName": "KVStore", "IndexName": "s_rating", "IndexStructure": "BTREE", "SearchKey": [ [ "area_code" ] ], "IsPrimary": false, "Timestamp": "Tue Aug 03 21:39:47 AST 2021", "PendingOp": 0, "ExcludeUnknownKey": false }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.04.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.04.adm
new file mode 100644
index 0000000..ff5b048
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.04.adm
@@ -0,0 +1,3 @@
+{ "values": [ 1, 1 ] }
+{ "values": [ 4, 4 ] }
+{ "values": [ 5, 5 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.05.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.05.adm
new file mode 100644
index 0000000..06d2398
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.05.adm
@@ -0,0 +1,5 @@
+{ "values": [ null, "2", 2 ] }
+{ "values": [ 1, "1", 1 ] }
+{ "values": [ 4, null, 4 ] }
+{ "values": [ 5, 5 ] }
+{ "values": [ "3", 3 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.06.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.06.adm
new file mode 100644
index 0000000..27e033c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.06.adm
@@ -0,0 +1,9 @@
+{ "values": [ null, 2 ] }
+{ "values": [ null, 6 ] }
+{ "values": [ null, 8 ] }
+{ "values": [ 1, 1 ] }
+{ "values": [ 3 ] }
+{ "values": [ 4, 4 ] }
+{ "values": [ 5, 5 ] }
+{ "values": [ 7 ] }
+{ "values": [ 9 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.07.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.07.adm
new file mode 100644
index 0000000..bdee019
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.07.adm
@@ -0,0 +1,9 @@
+{ "values": [ null, null, 8 ] }
+{ "values": [ null, 6 ] }
+{ "values": [ null, 7 ] }
+{ "values": [ null, "2", 2 ] }
+{ "values": [ 1, "1", 1 ] }
+{ "values": [ 4, null, 4 ] }
+{ "values": [ 5, 5 ] }
+{ "values": [ 9 ] }
+{ "values": [ "3", 3 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.08.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.08.adm
new file mode 100644
index 0000000..27e033c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.08.adm
@@ -0,0 +1,9 @@
+{ "values": [ null, 2 ] }
+{ "values": [ null, 6 ] }
+{ "values": [ null, 8 ] }
+{ "values": [ 1, 1 ] }
+{ "values": [ 3 ] }
+{ "values": [ 4, 4 ] }
+{ "values": [ 5, 5 ] }
+{ "values": [ 7 ] }
+{ "values": [ 9 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.09.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.09.adm
new file mode 100644
index 0000000..bdee019
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.09.adm
@@ -0,0 +1,9 @@
+{ "values": [ null, null, 8 ] }
+{ "values": [ null, 6 ] }
+{ "values": [ null, 7 ] }
+{ "values": [ null, "2", 2 ] }
+{ "values": [ 1, "1", 1 ] }
+{ "values": [ 4, null, 4 ] }
+{ "values": [ 5, 5 ] }
+{ "values": [ 9 ] }
+{ "values": [ "3", 3 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.10.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.10.adm
new file mode 100644
index 0000000..e839d15
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.10.adm
@@ -0,0 +1 @@
+{ "cnt": 9 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.11.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.11.adm
new file mode 100644
index 0000000..69fc309
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/index-unknown-key/index-unknown-key.11.adm
@@ -0,0 +1,7 @@
+{ "DataverseName": "test", "IndexName": "ds1" }
+{ "DataverseName": "test", "IndexName": "exclude_unknown_idx1", "ExcludeUnknownKey": true }
+{ "DataverseName": "test", "IndexName": "exclude_unknown_idx2", "ExcludeUnknownKey": true }
+{ "DataverseName": "test", "IndexName": "idx1", "ExcludeUnknownKey": false }
+{ "DataverseName": "test", "IndexName": "idx2", "ExcludeUnknownKey": false }
+{ "DataverseName": "test", "IndexName": "include_unknown_idx1", "ExcludeUnknownKey": false }
+{ "DataverseName": "test", "IndexName": "include_unknown_idx2", "ExcludeUnknownKey": false }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index a663fd3..2586447 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -5088,6 +5088,26 @@
<output-dir compare="Text">query-issue382</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="create-index-unknown-key">
+ <output-dir compare="Text">index-unknown-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="insert-into-index-unknown-key">
+ <output-dir compare="Text">index-unknown-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="upsert-into-index-unknown-key">
+ <output-dir compare="Text">index-unknown-key</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="load-into-index-unknown-key">
+ <output-dir compare="Text">index-unknown-key</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="employee">
<test-case FilePath="employee">
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
index 468006c..2c02812 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
@@ -48,10 +48,11 @@
private final int gramLength;
// Specific to FullText indexes.
private final String fullTextConfigName;
+ private final Boolean excludeUnknownKey;
public CreateIndexStatement(DataverseName dataverseName, Identifier datasetName, Identifier indexName,
IndexType indexType, List<IndexedElement> indexedElements, boolean enforced, int gramLength,
- String fullTextConfigName, boolean ifNotExists) {
+ String fullTextConfigName, boolean ifNotExists, Boolean excludeUnknownKey) {
this.dataverseName = dataverseName;
this.datasetName = Objects.requireNonNull(datasetName);
this.indexName = Objects.requireNonNull(indexName);
@@ -61,6 +62,7 @@
this.gramLength = gramLength;
this.ifNotExists = ifNotExists;
this.fullTextConfigName = fullTextConfigName;
+ this.excludeUnknownKey = excludeUnknownKey;
}
public String getFullTextConfigName() {
@@ -91,6 +93,14 @@
return enforced;
}
+ public boolean hasExcludeUnknownKey() {
+ return excludeUnknownKey != null;
+ }
+
+ public Boolean isExcludeUnknownKey() {
+ return excludeUnknownKey;
+ }
+
public int getGramLength() {
return gramLength;
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppStatementUtil.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppStatementUtil.java
index 5281dd9..7e1783b 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppStatementUtil.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppStatementUtil.java
@@ -32,6 +32,8 @@
public static final String CREATE_INDEX = "CREATE INDEX ";
public static final String CREATE_PRIMARY_INDEX = "CREATE PRIMARY INDEX ";
public static final String DROP_INDEX = "DROP INDEX ";
+ public static final String INCLUDE_UNKNOWN_KEY = " INCLUDE UNKNOWN KEY ";
+ public static final String EXCLUDE_UNKNOWN_KEY = " EXCLUDE UNKNOWN KEY ";
public static final String ON = " ON ";
public static final String WHERE = " WHERE ";
public static final String AND = " AND ";
@@ -69,10 +71,18 @@
@SuppressWarnings("squid:S1172") // unused variable
public static StringBuilder getCreateIndexStatement(StringBuilder stringBuilder, DataverseName dataverseName,
- String datasetName, String indexName, String fields, int version) {
+ String datasetName, String indexName, String fields, Boolean excludeUnknown, int version) {
stringBuilder.append(CREATE_INDEX);
enclose(stringBuilder, indexName).append(ON);
- return enclose(stringBuilder, dataverseName, datasetName).append(fields).append(SEMI_COLON);
+ StringBuilder appender = enclose(stringBuilder, dataverseName, datasetName).append(fields);
+ if (excludeUnknown != null) {
+ if (excludeUnknown) {
+ appender.append(EXCLUDE_UNKNOWN_KEY);
+ } else {
+ appender.append(INCLUDE_UNKNOWN_KEY);
+ }
+ }
+ return appender.append(SEMI_COLON);
}
@SuppressWarnings("squid:S1172") // unused variable
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index de9ea7a..b240779 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -227,6 +227,7 @@
private static final String CURRENT = "CURRENT";
private static final String DEFAULT = "DEFAULT";
private static final String EXCLUDE = "EXCLUDE";
+ private static final String INCLUDE = "INCLUDE";
private static final String FIRST = "FIRST";
private static final String FOLLOWING = "FOLLOWING";
private static final String GROUPING = "GROUPING";
@@ -1131,6 +1132,7 @@
boolean hasUnnest = false;
String fullTextConfigName = null;
Token startElementToken = null;
+ Boolean excludeUnknown = null;
}
{
(
@@ -1149,6 +1151,17 @@
)*
<RIGHTPAREN>
( <TYPE> indexParams = IndexType() )? ( <ENFORCED> { enforced = true; } )?
+ ( <IDENTIFIER>
+ {
+ if (isToken(EXCLUDE)) {
+ excludeUnknown = true;
+ } else if (isToken(INCLUDE)) {
+ excludeUnknown = false;
+ } else {
+ throw createUnexpectedTokenError();
+ }
+ } <UNKNOWN> <KEY>
+ )?
)
{
IndexType indexType;
@@ -1163,7 +1176,8 @@
fullTextConfigName = null;
}
CreateIndexStatement stmt = new CreateIndexStatement(nameComponents.first, nameComponents.second,
- new Identifier(indexName), indexType, indexedElementList, enforced, gramLength, fullTextConfigName, ifNotExists);
+ new Identifier(indexName), indexType, indexedElementList, enforced, gramLength, fullTextConfigName, ifNotExists,
+ excludeUnknown);
return addSourceLocation(stmt, startStmtToken);
}
}
@@ -1292,7 +1306,7 @@
indexName = "primary_idx_" + nameComponents.second;
}
CreateIndexStatement stmt = new CreateIndexStatement(nameComponents.first, nameComponents.second,
- new Identifier(indexName), IndexType.BTREE, Collections.emptyList(), false, -1, null, ifNotExists);
+ new Identifier(indexName), IndexType.BTREE, Collections.emptyList(), false, -1, null, ifNotExists, null);
return addSourceLocation(stmt, startStmtToken);
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index a5dda0d..87f5129 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -33,7 +33,6 @@
import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
-import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.dataflow.LSMIndexUtil;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.MetadataException;
@@ -364,9 +363,8 @@
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
// Add the primary index for the dataset.
InternalDatasetDetails id = (InternalDatasetDetails) dataset.getDatasetDetails();
- Index primaryIndex = new Index(dataset.getDataverseName(), dataset.getDatasetName(),
- dataset.getDatasetName(), IndexType.BTREE, id.getPrimaryKey(), id.getKeySourceIndicator(),
- id.getPrimaryKeyType(), false, false, true, dataset.getPendingOp());
+ Index primaryIndex = Index.createPrimaryIndex(dataset.getDataverseName(), dataset.getDatasetName(),
+ id.getPrimaryKey(), id.getKeySourceIndicator(), id.getPrimaryKeyType(), dataset.getPendingOp());
addIndex(txnId, primaryIndex);
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 6f91572..011d862 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -785,7 +785,7 @@
boolean bulkload, List<List<AlgebricksPipeline>> secondaryKeysPipelines, IOperatorSchema pipelineTopSchema)
throws AlgebricksException {
return getIndexInsertOrDeleteOrUpsertRuntime(IndexOperation.INSERT, dataSourceIndex, propagatedSchema,
- inputSchemas, typeEnv, primaryKeys, secondaryKeys, additionalNonKeyFields, filterExpr, recordDesc,
+ inputSchemas, typeEnv, primaryKeys, secondaryKeys, additionalNonKeyFields, filterExpr, null, recordDesc,
context, spec, bulkload, null, null, null, secondaryKeysPipelines, pipelineTopSchema);
}
@@ -798,7 +798,7 @@
List<List<AlgebricksPipeline>> secondaryKeysPipelines, IOperatorSchema pipelineTopSchema)
throws AlgebricksException {
return getIndexInsertOrDeleteOrUpsertRuntime(IndexOperation.DELETE, dataSourceIndex, propagatedSchema,
- inputSchemas, typeEnv, primaryKeys, secondaryKeys, additionalNonKeyFields, filterExpr, recordDesc,
+ inputSchemas, typeEnv, primaryKeys, secondaryKeys, additionalNonKeyFields, filterExpr, null, recordDesc,
context, spec, false, null, null, null, secondaryKeysPipelines, pipelineTopSchema);
}
@@ -807,12 +807,13 @@
IDataSourceIndex<String, DataSourceId> dataSourceIndex, IOperatorSchema propagatedSchema,
IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys,
List<LogicalVariable> secondaryKeys, List<LogicalVariable> additionalFilteringKeys,
- ILogicalExpression filterExpr, LogicalVariable operationVar, List<LogicalVariable> prevSecondaryKeys,
- LogicalVariable prevAdditionalFilteringKey, RecordDescriptor recordDesc, JobGenContext context,
- JobSpecification spec, List<List<AlgebricksPipeline>> secondaryKeysPipelines) throws AlgebricksException {
+ ILogicalExpression filterExpr, ILogicalExpression prevFilterExpr, LogicalVariable operationVar,
+ List<LogicalVariable> prevSecondaryKeys, LogicalVariable prevAdditionalFilteringKey,
+ RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec,
+ List<List<AlgebricksPipeline>> secondaryKeysPipelines) throws AlgebricksException {
return getIndexInsertOrDeleteOrUpsertRuntime(IndexOperation.UPSERT, dataSourceIndex, propagatedSchema,
- inputSchemas, typeEnv, primaryKeys, secondaryKeys, additionalFilteringKeys, filterExpr, recordDesc,
- context, spec, false, operationVar, prevSecondaryKeys, prevAdditionalFilteringKey,
+ inputSchemas, typeEnv, primaryKeys, secondaryKeys, additionalFilteringKeys, filterExpr, prevFilterExpr,
+ recordDesc, context, spec, false, operationVar, prevSecondaryKeys, prevAdditionalFilteringKey,
secondaryKeysPipelines, null);
}
@@ -1216,10 +1217,11 @@
IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv,
List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
List<LogicalVariable> additionalNonKeyFields, ILogicalExpression filterExpr,
- RecordDescriptor inputRecordDesc, JobGenContext context, JobSpecification spec, boolean bulkload,
- LogicalVariable operationVar, List<LogicalVariable> prevSecondaryKeys,
- LogicalVariable prevAdditionalFilteringKey, List<List<AlgebricksPipeline>> secondaryKeysPipelines,
- IOperatorSchema pipelineTopSchema) throws AlgebricksException {
+ ILogicalExpression prevFilterExpr, RecordDescriptor inputRecordDesc, JobGenContext context,
+ JobSpecification spec, boolean bulkload, LogicalVariable operationVar,
+ List<LogicalVariable> prevSecondaryKeys, LogicalVariable prevAdditionalFilteringKey,
+ List<List<AlgebricksPipeline>> secondaryKeysPipelines, IOperatorSchema pipelineTopSchema)
+ throws AlgebricksException {
String indexName = dataSourceIndex.getId();
DataverseName dataverseName = dataSourceIndex.getDataSource().getId().getDataverseName();
String datasetName = dataSourceIndex.getDataSource().getId().getDatasourceName();
@@ -1236,28 +1238,31 @@
// If we have a pipeline, then we need to pass the schema of the pipeline to the filter factory.
AsterixTupleFilterFactory filterFactory;
+ AsterixTupleFilterFactory prevFilterFactory;
if (pipelineTopSchema != null) {
IOperatorSchema[] schemasForFilterFactory = new IOperatorSchema[inputSchemas.length + 1];
System.arraycopy(inputSchemas, 0, schemasForFilterFactory, 0, inputSchemas.length);
schemasForFilterFactory[inputSchemas.length] = pipelineTopSchema;
filterFactory = createTupleFilterFactory(schemasForFilterFactory, typeEnv, filterExpr, context);
-
+ prevFilterFactory = createTupleFilterFactory(schemasForFilterFactory, typeEnv, prevFilterExpr, context);
} else {
filterFactory = createTupleFilterFactory(inputSchemas, typeEnv, filterExpr, context);
+ prevFilterFactory = createTupleFilterFactory(inputSchemas, typeEnv, prevFilterExpr, context);
}
switch (secondaryIndex.getIndexType()) {
case BTREE:
return getBTreeRuntime(dataverseName, datasetName, indexName, propagatedSchema, primaryKeys,
- secondaryKeys, additionalNonKeyFields, filterFactory, inputRecordDesc, context, spec, indexOp,
- bulkload, operationVar, prevSecondaryKeys, prevAdditionalFilteringKeys);
+ secondaryKeys, additionalNonKeyFields, filterFactory, prevFilterFactory, inputRecordDesc,
+ context, spec, indexOp, bulkload, operationVar, prevSecondaryKeys, prevAdditionalFilteringKeys);
case ARRAY:
if (bulkload) {
// In the case of bulk-load, we do not handle any nested plans. We perform the exact same behavior
// as a normal B-Tree bulk load.
return getBTreeRuntime(dataverseName, datasetName, indexName, propagatedSchema, primaryKeys,
- secondaryKeys, additionalNonKeyFields, filterFactory, inputRecordDesc, context, spec,
- indexOp, bulkload, operationVar, prevSecondaryKeys, prevAdditionalFilteringKeys);
+ secondaryKeys, additionalNonKeyFields, filterFactory, prevFilterFactory, inputRecordDesc,
+ context, spec, indexOp, bulkload, operationVar, prevSecondaryKeys,
+ prevAdditionalFilteringKeys);
} else {
return getArrayIndexRuntime(dataverseName, datasetName, indexName, propagatedSchema, primaryKeys,
additionalNonKeyFields, inputRecordDesc, spec, indexOp, operationVar,
@@ -1265,16 +1270,16 @@
}
case RTREE:
return getRTreeRuntime(dataverseName, datasetName, indexName, propagatedSchema, primaryKeys,
- secondaryKeys, additionalNonKeyFields, filterFactory, inputRecordDesc, context, spec, indexOp,
- bulkload, operationVar, prevSecondaryKeys, prevAdditionalFilteringKeys);
+ secondaryKeys, additionalNonKeyFields, filterFactory, prevFilterFactory, inputRecordDesc,
+ context, spec, indexOp, bulkload, operationVar, prevSecondaryKeys, prevAdditionalFilteringKeys);
case SINGLE_PARTITION_WORD_INVIX:
case SINGLE_PARTITION_NGRAM_INVIX:
case LENGTH_PARTITIONED_WORD_INVIX:
case LENGTH_PARTITIONED_NGRAM_INVIX:
return getInvertedIndexRuntime(dataverseName, datasetName, indexName, propagatedSchema, primaryKeys,
- secondaryKeys, additionalNonKeyFields, filterFactory, inputRecordDesc, context, spec, indexOp,
- secondaryIndex.getIndexType(), bulkload, operationVar, prevSecondaryKeys,
- prevAdditionalFilteringKeys);
+ secondaryKeys, additionalNonKeyFields, filterFactory, prevFilterFactory, inputRecordDesc,
+ context, spec, indexOp, secondaryIndex.getIndexType(), bulkload, operationVar,
+ prevSecondaryKeys, prevAdditionalFilteringKeys);
default:
throw new AlgebricksException(
indexOp.name() + " not implemented for index type: " + secondaryIndex.getIndexType());
@@ -1284,10 +1289,10 @@
private Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getBTreeRuntime(DataverseName dataverseName,
String datasetName, String indexName, IOperatorSchema propagatedSchema, List<LogicalVariable> primaryKeys,
List<LogicalVariable> secondaryKeys, List<LogicalVariable> additionalNonKeyFields,
- AsterixTupleFilterFactory filterFactory, RecordDescriptor inputRecordDesc, JobGenContext context,
- JobSpecification spec, IndexOperation indexOp, boolean bulkload, LogicalVariable operationVar,
- List<LogicalVariable> prevSecondaryKeys, List<LogicalVariable> prevAdditionalFilteringKeys)
- throws AlgebricksException {
+ AsterixTupleFilterFactory filterFactory, AsterixTupleFilterFactory prevFilterFactory,
+ RecordDescriptor inputRecordDesc, JobGenContext context, JobSpecification spec, IndexOperation indexOp,
+ boolean bulkload, LogicalVariable operationVar, List<LogicalVariable> prevSecondaryKeys,
+ List<LogicalVariable> prevAdditionalFilteringKeys) throws AlgebricksException {
Dataset dataset = MetadataManagerUtil.findExistingDataset(mdTxnCtx, dataverseName, datasetName);
int numKeys = primaryKeys.size() + secondaryKeys.size();
int numFilterFields = DatasetUtil.getFilterField(dataset) == null ? 0 : 1;
@@ -1356,8 +1361,8 @@
} else if (indexOp == IndexOperation.UPSERT) {
int operationFieldIndex = propagatedSchema.findVariable(operationVar);
op = new LSMSecondaryUpsertOperatorDescriptor(spec, inputRecordDesc, fieldPermutation, idfh,
- filterFactory, modificationCallbackFactory, operationFieldIndex, BinaryIntegerInspector.FACTORY,
- prevFieldPermutation);
+ filterFactory, prevFilterFactory, modificationCallbackFactory, operationFieldIndex,
+ BinaryIntegerInspector.FACTORY, prevFieldPermutation);
} else {
op = new LSMTreeInsertDeleteOperatorDescriptor(spec, inputRecordDesc, fieldPermutation, indexOp, idfh,
filterFactory, false, modificationCallbackFactory);
@@ -1425,10 +1430,10 @@
private Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getRTreeRuntime(DataverseName dataverseName,
String datasetName, String indexName, IOperatorSchema propagatedSchema, List<LogicalVariable> primaryKeys,
List<LogicalVariable> secondaryKeys, List<LogicalVariable> additionalNonKeyFields,
- AsterixTupleFilterFactory filterFactory, RecordDescriptor recordDesc, JobGenContext context,
- JobSpecification spec, IndexOperation indexOp, boolean bulkload, LogicalVariable operationVar,
- List<LogicalVariable> prevSecondaryKeys, List<LogicalVariable> prevAdditionalFilteringKeys)
- throws AlgebricksException {
+ AsterixTupleFilterFactory filterFactory, AsterixTupleFilterFactory prevFilterFactory,
+ RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec, IndexOperation indexOp,
+ boolean bulkload, LogicalVariable operationVar, List<LogicalVariable> prevSecondaryKeys,
+ List<LogicalVariable> prevAdditionalFilteringKeys) throws AlgebricksException {
Dataset dataset = MetadataManagerUtil.findExistingDataset(mdTxnCtx, dataverseName, datasetName);
String itemTypeName = dataset.getItemTypeName();
IAType itemType = MetadataManager.INSTANCE
@@ -1511,8 +1516,8 @@
} else if (indexOp == IndexOperation.UPSERT) {
int operationFieldIndex = propagatedSchema.findVariable(operationVar);
op = new LSMSecondaryUpsertOperatorDescriptor(spec, recordDesc, fieldPermutation,
- indexDataflowHelperFactory, filterFactory, modificationCallbackFactory, operationFieldIndex,
- BinaryIntegerInspector.FACTORY, prevFieldPermutation);
+ indexDataflowHelperFactory, filterFactory, prevFilterFactory, modificationCallbackFactory,
+ operationFieldIndex, BinaryIntegerInspector.FACTORY, prevFieldPermutation);
} else {
op = new LSMTreeInsertDeleteOperatorDescriptor(spec, recordDesc, fieldPermutation, indexOp,
indexDataflowHelperFactory, filterFactory, false, modificationCallbackFactory);
@@ -1524,10 +1529,10 @@
DataverseName dataverseName, String datasetName, String indexName, IOperatorSchema propagatedSchema,
List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
List<LogicalVariable> additionalNonKeyFields, AsterixTupleFilterFactory filterFactory,
- RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec, IndexOperation indexOp,
- IndexType indexType, boolean bulkload, LogicalVariable operationVar,
- List<LogicalVariable> prevSecondaryKeys, List<LogicalVariable> prevAdditionalFilteringKeys)
- throws AlgebricksException {
+ AsterixTupleFilterFactory prevFilterFactory, RecordDescriptor recordDesc, JobGenContext context,
+ JobSpecification spec, IndexOperation indexOp, IndexType indexType, boolean bulkload,
+ LogicalVariable operationVar, List<LogicalVariable> prevSecondaryKeys,
+ List<LogicalVariable> prevAdditionalFilteringKeys) throws AlgebricksException {
// Check the index is length-partitioned or not.
boolean isPartitioned;
if (indexType == IndexType.LENGTH_PARTITIONED_WORD_INVIX
@@ -1624,7 +1629,7 @@
} else if (indexOp == IndexOperation.UPSERT) {
int upsertOperationFieldIndex = propagatedSchema.findVariable(operationVar);
op = new LSMSecondaryUpsertOperatorDescriptor(spec, recordDesc, fieldPermutation, indexDataFlowFactory,
- filterFactory, modificationCallbackFactory, upsertOperationFieldIndex,
+ filterFactory, prevFilterFactory, modificationCallbackFactory, upsertOperationFieldIndex,
BinaryIntegerInspector.FACTORY, prevFieldPermutation);
} else {
op = new LSMTreeInsertDeleteOperatorDescriptor(spec, recordDesc, fieldPermutation, indexOp,
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
index cf09779..685dc85 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
@@ -81,13 +81,23 @@
@Deprecated
public Index(DataverseName dataverseName, String datasetName, String indexName, IndexType indexType,
List<List<String>> keyFieldNames, List<Integer> keyFieldSourceIndicators, List<IAType> keyFieldTypes,
- boolean overrideKeyFieldTypes, boolean isEnforced, boolean isPrimaryIndex, int pendingOp) {
- this(dataverseName,
- datasetName, indexName, indexType, createSimpleIndexDetails(indexType, keyFieldNames,
- keyFieldSourceIndicators, keyFieldTypes, overrideKeyFieldTypes),
+ boolean overrideKeyFieldTypes, boolean isEnforced, boolean isPrimaryIndex, int pendingOp,
+ Boolean excludeUnknownKey) {
+ this(dataverseName, datasetName,
+ indexName, indexType, createSimpleIndexDetails(indexType, keyFieldNames, keyFieldSourceIndicators,
+ keyFieldTypes, overrideKeyFieldTypes, excludeUnknownKey),
isEnforced, isPrimaryIndex, pendingOp);
}
+ public static Index createPrimaryIndex(DataverseName dataverseName, String datasetName,
+ List<List<String>> keyFieldNames, List<Integer> keyFieldSourceIndicators, List<IAType> keyFieldTypes,
+ int pendingOp) {
+ return new Index(dataverseName, datasetName, datasetName, IndexType.BTREE,
+ createSimpleIndexDetails(IndexType.BTREE, keyFieldNames, keyFieldSourceIndicators, keyFieldTypes, false,
+ null),
+ false, true, pendingOp);
+ }
+
public DataverseName getDataverseName() {
return dataverseName;
}
@@ -322,12 +332,15 @@
private final boolean overrideKeyFieldTypes;
+ private final Boolean excludeUnknownKey;
+
public ValueIndexDetails(List<List<String>> keyFieldNames, List<Integer> keyFieldSourceIndicators,
- List<IAType> keyFieldTypes, boolean overrideKeyFieldTypes) {
+ List<IAType> keyFieldTypes, boolean overrideKeyFieldTypes, Boolean excludeUnknownKey) {
this.keyFieldNames = keyFieldNames;
this.keyFieldSourceIndicators = keyFieldSourceIndicators;
this.keyFieldTypes = keyFieldTypes;
this.overrideKeyFieldTypes = overrideKeyFieldTypes;
+ this.excludeUnknownKey = excludeUnknownKey;
}
@Override
@@ -347,6 +360,10 @@
return keyFieldTypes;
}
+ public Boolean isExcludeUnknownKey() {
+ return excludeUnknownKey;
+ }
+
@Override
public boolean isOverridingKeyFieldTypes() {
return overrideKeyFieldTypes;
@@ -481,14 +498,15 @@
@Deprecated
private static Index.IIndexDetails createSimpleIndexDetails(IndexType indexType, List<List<String>> keyFieldNames,
- List<Integer> keyFieldSourceIndicators, List<IAType> keyFieldTypes, boolean overrideKeyFieldTypes) {
+ List<Integer> keyFieldSourceIndicators, List<IAType> keyFieldTypes, boolean overrideKeyFieldTypes,
+ Boolean excludeUnknownKey) {
if (indexType == null) {
return null;
}
switch (Index.IndexCategory.of(indexType)) {
case VALUE:
return new ValueIndexDetails(keyFieldNames, keyFieldSourceIndicators, keyFieldTypes,
- overrideKeyFieldTypes);
+ overrideKeyFieldTypes, excludeUnknownKey);
case TEXT:
return new TextIndexDetails(keyFieldNames, keyFieldSourceIndicators, keyFieldTypes,
overrideKeyFieldTypes, -1, null);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
index 32764b2..3c322a4 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
@@ -81,6 +81,7 @@
public static final String FULL_TEXT_CONFIG_FIELD_NAME = "FullTextConfig";
public static final String INDEX_SEARCHKEY_TYPE_FIELD_NAME = "SearchKeyType";
public static final String INDEX_ISENFORCED_FIELD_NAME = "IsEnforced";
+ public static final String INDEX_EXCLUDE_UNKNOWN_FIELD_NAME = "ExcludeUnknownKey";
public static final String INDEX_SEARCHKEY_SOURCE_INDICATOR_FIELD_NAME = "SearchKeySourceIndicator";
public static final String INDEX_SEARCHKEY_ELEMENTS_FIELD_NAME = "SearchKeyElements";
public static final String COMPLEXSEARCHKEY_UNNEST_FIELD_NAME = "UnnestList";
@@ -138,6 +139,9 @@
IndexType indexType = IndexType.valueOf(
((AString) indexRecord.getValueByPos(MetadataRecordTypes.INDEX_ARECORD_INDEXSTRUCTURE_FIELD_INDEX))
.getStringValue());
+ Boolean isPrimaryIndex =
+ ((ABoolean) indexRecord.getValueByPos(MetadataRecordTypes.INDEX_ARECORD_ISPRIMARY_FIELD_INDEX))
+ .getBoolean();
// Read key names
List<Pair<List<List<String>>, List<List<String>>>> searchElements = new ArrayList<>();
@@ -377,8 +381,21 @@
List<List<String>> keyFieldNames =
searchElements.stream().map(Pair::getSecond).map(l -> l.get(0)).collect(Collectors.toList());
List<IAType> keyFieldTypes = searchKeyType.stream().map(l -> l.get(0)).collect(Collectors.toList());
+
+ // Read the exclude unknown key option if applicable for an index
+ Boolean excludeUnknownKey = null;
+ boolean unknownKeyOptionAllowed =
+ indexType == IndexType.BTREE && !isPrimaryIndex && !keyFieldNames.isEmpty();
+ if (unknownKeyOptionAllowed) {
+ // default to always include unknowns for normal b-trees
+ excludeUnknownKey = false;
+ int excludeUnknownKeyPos = indexRecord.getType().getFieldIndex(INDEX_EXCLUDE_UNKNOWN_FIELD_NAME);
+ if (excludeUnknownKeyPos >= 0) {
+ excludeUnknownKey = ((ABoolean) indexRecord.getValueByPos(excludeUnknownKeyPos)).getBoolean();
+ }
+ }
indexDetails = new Index.ValueIndexDetails(keyFieldNames, keyFieldSourceIndicator, keyFieldTypes,
- isOverridingKeyTypes);
+ isOverridingKeyTypes, excludeUnknownKey);
break;
case TEXT:
keyFieldNames =
@@ -413,9 +430,6 @@
if (isEnforcedFieldPos > 0) {
isEnforcingKeys = ((ABoolean) indexRecord.getValueByPos(isEnforcedFieldPos)).getBoolean();
}
- Boolean isPrimaryIndex =
- ((ABoolean) indexRecord.getValueByPos(MetadataRecordTypes.INDEX_ARECORD_ISPRIMARY_FIELD_INDEX))
- .getBoolean();
int pendingOp = ((AInt32) indexRecord.getValueByPos(MetadataRecordTypes.INDEX_ARECORD_PENDINGOP_FIELD_INDEX))
.getIntegerValue();
@@ -549,6 +563,7 @@
writeSearchKeyType(index);
writeEnforced(index);
writeSearchKeySourceIndicator(index);
+ writeExcludeUnknownKey(index);
}
private void writeComplexSearchKeys(Index.ArrayIndexDetails indexDetails) throws HyracksDataException {
@@ -748,4 +763,21 @@
recordBuilder.addField(nameValue, fieldValue);
}
}
+
+ private void writeExcludeUnknownKey(Index index) throws HyracksDataException {
+ boolean unknownKeyOptionAllowed =
+ index.getIndexType() == IndexType.BTREE && !index.isPrimaryIndex() && !index.isPrimaryKeyIndex();
+ if (unknownKeyOptionAllowed) {
+ Boolean excludeUnknownKey = ((Index.ValueIndexDetails) index.getIndexDetails()).isExcludeUnknownKey();
+ if (excludeUnknownKey == null) {
+ excludeUnknownKey = false;
+ }
+ fieldValue.reset();
+ nameValue.reset();
+ aString.setValue(INDEX_EXCLUDE_UNKNOWN_FIELD_NAME);
+ stringSerde.serialize(aString, nameValue.getDataOutput());
+ booleanSerde.serialize(ABoolean.valueOf(excludeUnknownKey), fieldValue.getDataOutput());
+ recordBuilder.addField(nameValue, fieldValue);
+ }
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
index f5b2697..f15502f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
@@ -58,9 +58,8 @@
public static Index getPrimaryIndex(Dataset dataset) {
InternalDatasetDetails id = (InternalDatasetDetails) dataset.getDatasetDetails();
- return new Index(dataset.getDataverseName(), dataset.getDatasetName(), dataset.getDatasetName(),
- DatasetConfig.IndexType.BTREE, id.getPartitioningKey(), id.getKeySourceIndicator(),
- id.getPrimaryKeyType(), false, false, true, dataset.getPendingOp());
+ return Index.createPrimaryIndex(dataset.getDataverseName(), dataset.getDatasetName(), id.getPartitioningKey(),
+ id.getKeySourceIndicator(), id.getPrimaryKeyType(), dataset.getPendingOp());
}
public static int[] getBtreeFieldsIfFiltered(Dataset dataset, Index index) throws AlgebricksException {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
index c1f8c7b..8ce63f1 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
@@ -286,7 +286,7 @@
if (anySecondaryKeyIsNullable || arrayIndexDetails.isOverridingKeyFieldTypes()) {
// If any of the secondary fields are nullable, then we need to filter out the nulls.
- targetOp = createFilterNullsSelectOp(spec, numTotalSecondaryKeys, secondaryRecDesc);
+ targetOp = createFilterAnyUnknownSelectOp(spec, numTotalSecondaryKeys, secondaryRecDesc);
spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
sourceOp = targetOp;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
index 9645e7a..5a1a06e 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
@@ -70,6 +70,8 @@
int[] fieldPermutation = createFieldPermutationForBulkLoadOp(indexDetails.getKeyFieldNames().size());
IIndexDataflowHelperFactory dataflowHelperFactory = new IndexDataflowHelperFactory(
metadataProvider.getStorageComponentProvider().getStorageManager(), secondaryFileSplitProvider);
+ boolean excludeUnknown =
+ excludeUnknowns(index, indexDetails) && (anySecondaryKeyIsNullable || isOverridingKeyFieldTypes);
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
/*
* In case of external data,
@@ -89,6 +91,12 @@
AlgebricksMetaOperatorDescriptor asterixAssignOp =
createExternalAssignOp(spec, indexDetails.getKeyFieldNames().size(), secondaryRecDesc);
+ // If any of the secondary fields are nullable, then add a select op that filters nulls.
+ AlgebricksMetaOperatorDescriptor selectOp = null;
+ if (excludeUnknown) {
+ selectOp =
+ createFilterAllUnknownsSelectOp(spec, indexDetails.getKeyFieldNames().size(), secondaryRecDesc);
+ }
// Sort by secondary keys.
ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
// Create secondary BTree bulk load op.
@@ -111,7 +119,12 @@
spec.connect(new OneToOneConnectorDescriptor(spec), secondaryBulkLoadOp, 0, metaOp, 0);
root = metaOp;
spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, asterixAssignOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
+ if (excludeUnknown) {
+ spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0);
+ } else {
+ spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
+ }
spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
spec.addRoot(root);
spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
@@ -138,7 +151,14 @@
spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
sourceOp = targetOp;
-
+ if (excludeUnknown) {
+ // if any of the secondary fields are nullable, then add a select op that filters nulls.
+ // assign op ----> select op
+ targetOp =
+ createFilterAllUnknownsSelectOp(spec, indexDetails.getKeyFieldNames().size(), secondaryRecDesc);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
+ sourceOp = targetOp;
+ }
// no need to sort if the index is secondary primary index
if (!indexDetails.getKeyFieldNames().isEmpty()) {
// sort by secondary keys.
@@ -288,4 +308,13 @@
}
return fieldPermutation;
}
+
+ private static boolean excludeUnknowns(Index index, Index.ValueIndexDetails details) {
+ if (index.isPrimaryKeyIndex()) {
+ return true;
+ } else {
+ Boolean excludeUnknownKey = details.isExcludeUnknownKey();
+ return excludeUnknownKey != null && excludeUnknownKey;
+ }
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index a6e3087..b318fe2 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -23,6 +23,7 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.Supplier;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
@@ -43,6 +44,7 @@
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.metadata.lock.ExternalDatasetsRegistry;
+import org.apache.asterix.om.functions.AbstractFunctionDescriptor;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.functions.IFunctionDescriptor;
import org.apache.asterix.om.types.ARecordType;
@@ -50,6 +52,7 @@
import org.apache.asterix.runtime.evaluators.functions.AndDescriptor;
import org.apache.asterix.runtime.evaluators.functions.IsUnknownDescriptor;
import org.apache.asterix.runtime.evaluators.functions.NotDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.OrDescriptor;
import org.apache.asterix.runtime.operators.LSMIndexBulkLoadOperatorDescriptor;
import org.apache.asterix.runtime.operators.LSMIndexBulkLoadOperatorDescriptor.BulkLoadUsage;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
@@ -386,9 +389,20 @@
return treeIndexBulkLoadOp;
}
- public AlgebricksMetaOperatorDescriptor createFilterNullsSelectOp(JobSpecification spec, int numSecondaryKeyFields,
- RecordDescriptor secondaryRecDesc) {
- IScalarEvaluatorFactory[] andArgsEvalFactories = new IScalarEvaluatorFactory[numSecondaryKeyFields];
+ public AlgebricksMetaOperatorDescriptor createFilterAllUnknownsSelectOp(JobSpecification spec,
+ int numSecondaryKeyFields, RecordDescriptor secondaryRecDesc) throws AlgebricksException {
+ return createFilterSelectOp(spec, numSecondaryKeyFields, secondaryRecDesc, OrDescriptor::new);
+ }
+
+ public AlgebricksMetaOperatorDescriptor createFilterAnyUnknownSelectOp(JobSpecification spec,
+ int numSecondaryKeyFields, RecordDescriptor secondaryRecDesc) throws AlgebricksException {
+ return createFilterSelectOp(spec, numSecondaryKeyFields, secondaryRecDesc, AndDescriptor::new);
+ }
+
+ private AlgebricksMetaOperatorDescriptor createFilterSelectOp(JobSpecification spec, int numSecondaryKeyFields,
+ RecordDescriptor secondaryRecDesc, Supplier<AbstractFunctionDescriptor> predicatesCombinerFuncSupplier)
+ throws AlgebricksException {
+ IScalarEvaluatorFactory[] predicateArgsEvalFactories = new IScalarEvaluatorFactory[numSecondaryKeyFields];
NotDescriptor notDesc = new NotDescriptor();
notDesc.setSourceLocation(sourceLoc);
IsUnknownDescriptor isUnknownDesc = new IsUnknownDescriptor();
@@ -400,17 +414,15 @@
isUnknownDesc.createEvaluatorFactory(new IScalarEvaluatorFactory[] { columnAccessEvalFactory });
IScalarEvaluatorFactory notEvalFactory =
notDesc.createEvaluatorFactory(new IScalarEvaluatorFactory[] { isUnknownEvalFactory });
- andArgsEvalFactories[i] = notEvalFactory;
+ predicateArgsEvalFactories[i] = notEvalFactory;
}
IScalarEvaluatorFactory selectCond;
if (numSecondaryKeyFields > 1) {
- // Create conjunctive condition where all secondary index keys must
- // satisfy 'is not null'.
- AndDescriptor andDesc = new AndDescriptor();
- andDesc.setSourceLocation(sourceLoc);
- selectCond = andDesc.createEvaluatorFactory(andArgsEvalFactories);
+ AbstractFunctionDescriptor predicatesCombiner = predicatesCombinerFuncSupplier.get();
+ predicatesCombiner.setSourceLocation(sourceLoc);
+ selectCond = predicatesCombiner.createEvaluatorFactory(predicateArgsEvalFactories);
} else {
- selectCond = andArgsEvalFactories[0];
+ selectCond = predicateArgsEvalFactories[0];
}
StreamSelectRuntimeFactory select =
new StreamSelectRuntimeFactory(selectCond, null, BinaryBooleanInspector.FACTORY, false, -1, null);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
index 2d5a4f8..67f58a5 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
@@ -236,7 +236,7 @@
// that filters nulls.
AlgebricksMetaOperatorDescriptor selectOp = null;
if (anySecondaryKeyIsNullable || isOverridingKeyFieldTypes) {
- selectOp = createFilterNullsSelectOp(spec, numSecondaryKeys, secondaryRecDesc);
+ selectOp = createFilterAnyUnknownSelectOp(spec, numSecondaryKeys, secondaryRecDesc);
}
// Create a tokenizer op.
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
index dbbd723..e4b5cf8 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
@@ -222,7 +222,7 @@
// If any of the secondary fields are nullable, then add a select op that filters nulls.
AlgebricksMetaOperatorDescriptor selectOp = null;
if (anySecondaryKeyIsNullable || isOverridingKeyFieldTypes) {
- selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeFieldsConsideringPointMBR,
+ selectOp = createFilterAnyUnknownSelectOp(spec, numNestedSecondaryKeFieldsConsideringPointMBR,
secondaryRecDescConsideringPointMBR);
}
@@ -273,7 +273,7 @@
// If any of the secondary fields are nullable, then add a select op that filters nulls.
AlgebricksMetaOperatorDescriptor selectOp = null;
if (anySecondaryKeyIsNullable || isOverridingKeyFieldTypes) {
- selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeFieldsConsideringPointMBR,
+ selectOp = createFilterAnyUnknownSelectOp(spec, numNestedSecondaryKeFieldsConsideringPointMBR,
secondaryRecDescConsideringPointMBR);
}
diff --git a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java
index f690018..ad04bbe 100644
--- a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java
+++ b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java
@@ -73,7 +73,7 @@
Index index = new Index(dvTest, "d1", "i1", IndexType.BTREE,
Collections.singletonList(Collections.singletonList("row_id")),
indicator == null ? null : Collections.singletonList(indicator),
- Collections.singletonList(BuiltinType.AINT64), false, false, false, 0);
+ Collections.singletonList(BuiltinType.AINT64), false, false, false, 0, false);
MetadataNode mockMetadataNode = mock(MetadataNode.class);
when(mockMetadataNode.getDatatype(any(), any(DataverseName.class), anyString())).thenReturn(new Datatype(
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertOperatorDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertOperatorDescriptor.java
index 3231162..9de0827 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertOperatorDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertOperatorDescriptor.java
@@ -33,21 +33,23 @@
public class LSMSecondaryUpsertOperatorDescriptor extends LSMTreeInsertDeleteOperatorDescriptor {
- private static final long serialVersionUID = 2L;
+ private static final long serialVersionUID = 3L;
private final int[] prevValuePermutation;
protected final int operationFieldIndex;
protected final IBinaryIntegerInspectorFactory operationInspectorFactory;
+ private final ITupleFilterFactory prevTupleFilterFactory;
public LSMSecondaryUpsertOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor outRecDesc,
int[] fieldPermutation, IIndexDataflowHelperFactory indexHelperFactory,
- ITupleFilterFactory tupleFilterFactory, IModificationOperationCallbackFactory modificationOpCallbackFactory,
- int operationFieldIndex, IBinaryIntegerInspectorFactory operationInspectorFactory,
- int[] prevValuePermutation) {
+ ITupleFilterFactory tupleFilterFactory, ITupleFilterFactory prevTupleFilterFactory,
+ IModificationOperationCallbackFactory modificationOpCallbackFactory, int operationFieldIndex,
+ IBinaryIntegerInspectorFactory operationInspectorFactory, int[] prevValuePermutation) {
super(spec, outRecDesc, fieldPermutation, IndexOperation.UPSERT, indexHelperFactory, tupleFilterFactory, false,
modificationOpCallbackFactory);
this.prevValuePermutation = prevValuePermutation;
this.operationFieldIndex = operationFieldIndex;
this.operationInspectorFactory = operationInspectorFactory;
+ this.prevTupleFilterFactory = prevTupleFilterFactory;
}
@Override
@@ -55,7 +57,7 @@
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
RecordDescriptor intputRecDesc = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);
return new LSMSecondaryUpsertOperatorNodePushable(ctx, partition, indexHelperFactory, modCallbackFactory,
- tupleFilterFactory, fieldPermutation, intputRecDesc, operationFieldIndex, operationInspectorFactory,
- prevValuePermutation);
+ tupleFilterFactory, prevTupleFilterFactory, fieldPermutation, intputRecDesc, operationFieldIndex,
+ operationInspectorFactory, prevValuePermutation);
}
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertOperatorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertOperatorNodePushable.java
index 482be06..955d5aa 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertOperatorNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertOperatorNodePushable.java
@@ -34,6 +34,7 @@
import org.apache.hyracks.dataflow.common.data.accessors.PermutingFrameTupleReference;
import org.apache.hyracks.dataflow.common.utils.TupleUtils;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.api.ITupleFilter;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
@@ -57,6 +58,8 @@
private final PermutingFrameTupleReference prevTuple = new PermutingFrameTupleReference();
private final int numberOfFields;
+ private final ITupleFilterFactory prevTupleFilterFactory;
+ private ITupleFilter prevTupleFilter;
protected final int operationFieldIndex;
protected final IBinaryIntegerInspector operationInspector;
@@ -64,15 +67,18 @@
public LSMSecondaryUpsertOperatorNodePushable(IHyracksTaskContext ctx, int partition,
IIndexDataflowHelperFactory indexHelperFactory, IModificationOperationCallbackFactory modCallbackFactory,
- ITupleFilterFactory tupleFilterFactory, int[] fieldPermutation, RecordDescriptor inputRecDesc,
- int operationFieldIndex, IBinaryIntegerInspectorFactory operationInspectorFactory,
- int[] prevTuplePermutation) throws HyracksDataException {
+ ITupleFilterFactory tupleFilterFactory, ITupleFilterFactory prevTupleFilterFactory, int[] fieldPermutation,
+ RecordDescriptor inputRecDesc, int operationFieldIndex,
+ IBinaryIntegerInspectorFactory operationInspectorFactory, int[] prevTuplePermutation)
+ throws HyracksDataException {
super(ctx, partition, indexHelperFactory, fieldPermutation, inputRecDesc, IndexOperation.UPSERT,
modCallbackFactory, tupleFilterFactory);
this.prevTuple.setFieldPermutation(prevTuplePermutation);
this.operationFieldIndex = operationFieldIndex;
this.operationInspector = operationInspectorFactory.createBinaryIntegerInspector(ctx);
this.numberOfFields = fieldPermutation.length;
+ this.prevTupleFilterFactory = prevTupleFilterFactory;
+
}
@Override
@@ -80,6 +86,9 @@
super.open();
frameTuple = new FrameTupleReference();
abstractModCallback = (AbstractIndexModificationOperationCallback) modCallback;
+ if (prevTupleFilterFactory != null) {
+ prevTupleFilter = prevTupleFilterFactory.createTupleFilter(ctx);
+ }
}
@Override
@@ -87,6 +96,8 @@
accessor.reset(buffer);
ILSMIndexAccessor lsmAccessor = (ILSMIndexAccessor) indexAccessor;
int tupleCount = accessor.getTupleCount();
+ boolean tupleFilterIsNull = tupleFilter == null;
+ boolean prevTupleFilterIsNull = prevTupleFilter == null;
for (int i = 0; i < tupleCount; i++) {
try {
frameTuple.reset(accessor, i);
@@ -96,18 +107,26 @@
prevTuple.reset(accessor, i);
if (operation == UPSERT_NEW) {
- abstractModCallback.setOp(Operation.INSERT);
- lsmAccessor.forceInsert(tuple);
- } else if (operation == UPSERT_EXISTING) {
- if (!TupleUtils.equalTuples(tuple, prevTuple, numberOfFields)) {
- abstractModCallback.setOp(Operation.DELETE);
- lsmAccessor.forceDelete(prevTuple);
+ if (tupleFilterIsNull || tupleFilter.accept(frameTuple)) {
abstractModCallback.setOp(Operation.INSERT);
lsmAccessor.forceInsert(tuple);
}
+ } else if (operation == UPSERT_EXISTING) {
+ if (!TupleUtils.equalTuples(tuple, prevTuple, numberOfFields)) {
+ if (prevTupleFilterIsNull || prevTupleFilter.accept(frameTuple)) {
+ abstractModCallback.setOp(Operation.DELETE);
+ lsmAccessor.forceDelete(prevTuple);
+ }
+ if (tupleFilterIsNull || tupleFilter.accept(frameTuple)) {
+ abstractModCallback.setOp(Operation.INSERT);
+ lsmAccessor.forceInsert(tuple);
+ }
+ }
} else if (operation == DELETE_EXISTING) {
- abstractModCallback.setOp(Operation.DELETE);
- lsmAccessor.forceDelete(prevTuple);
+ if (prevTupleFilterIsNull || prevTupleFilter.accept(frameTuple)) {
+ abstractModCallback.setOp(Operation.DELETE);
+ lsmAccessor.forceDelete(prevTuple);
+ }
}
} catch (Exception e) {
throw HyracksDataException.create(e);
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertWithNestedPlanOperatorDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertWithNestedPlanOperatorDescriptor.java
index bbf0af1..41bd0fb 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertWithNestedPlanOperatorDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertWithNestedPlanOperatorDescriptor.java
@@ -42,8 +42,8 @@
IModificationOperationCallbackFactory modCallbackFactory, int operationFieldIndex,
IBinaryIntegerInspectorFactory operationInspectorFactory, List<AlgebricksPipeline> secondaryKeysPipeline,
List<AlgebricksPipeline> prevSecondaryKeysPipeline) {
- super(spec, outRecDesc, fieldPermutation, indexHelperFactory, null, modCallbackFactory, operationFieldIndex,
- operationInspectorFactory, null);
+ super(spec, outRecDesc, fieldPermutation, indexHelperFactory, null, null, modCallbackFactory,
+ operationFieldIndex, operationInspectorFactory, null);
this.secondaryKeysPipeline = secondaryKeysPipeline;
this.prevSecondaryKeysPipeline = prevSecondaryKeysPipeline;
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertWithNestedPlanOperatorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertWithNestedPlanOperatorNodePushable.java
index 3fe3e85..08fd566 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertWithNestedPlanOperatorNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertWithNestedPlanOperatorNodePushable.java
@@ -51,7 +51,7 @@
int[] fieldPermutation, RecordDescriptor inputRecDesc, int operationFieldIndex,
IBinaryIntegerInspectorFactory operationInspectorFactory, List<AlgebricksPipeline> secondaryKeysPipeline,
List<AlgebricksPipeline> prevSecondaryKeysPipeline) throws HyracksDataException {
- super(ctx, partition, indexHelperFactory, modCallbackFactory, null, fieldPermutation, inputRecDesc,
+ super(ctx, partition, indexHelperFactory, modCallbackFactory, null, null, fieldPermutation, inputRecDesc,
operationFieldIndex, operationInspectorFactory, null);
this.numberOfPrimaryKeyAndFilterFields = fieldPermutation.length;
this.startOfNewKeyPipelines = buildStartOfPipelines(secondaryKeysPipeline, inputRecDesc, false);
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
index 9bafe3e..77fdd74 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
@@ -218,10 +218,10 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexUpsertRuntime(
IDataSourceIndex<I, S> dataSourceIndex, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
- List<LogicalVariable> additionalFilteringKeys, ILogicalExpression filterExpr, LogicalVariable operationVar,
- List<LogicalVariable> prevSecondaryKeys, LogicalVariable prevAdditionalFilteringKeys,
- RecordDescriptor inputDesc, JobGenContext context, JobSpecification spec,
- List<List<AlgebricksPipeline>> secondaryKeysPipelines) throws AlgebricksException;
+ List<LogicalVariable> additionalFilteringKeys, ILogicalExpression filterExpr,
+ ILogicalExpression prevFilterExpr, LogicalVariable operationVar, List<LogicalVariable> prevSecondaryKeys,
+ LogicalVariable prevAdditionalFilteringKeys, RecordDescriptor inputDesc, JobGenContext context,
+ JobSpecification spec, List<List<AlgebricksPipeline>> secondaryKeysPipelines) throws AlgebricksException;
public ITupleFilterFactory createTupleFilterFactory(IOperatorSchema[] inputSchemas,
IVariableTypeEnvironment typeEnv, ILogicalExpression filterExpr, JobGenContext context)
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteUpsertOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteUpsertOperator.java
index f2ac41a..0098817 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteUpsertOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteUpsertOperator.java
@@ -78,6 +78,7 @@
// Otherwise, it contains secondary key information.
private List<Mutable<ILogicalExpression>> secondaryKeyExprs;
private Mutable<ILogicalExpression> filterExpr;
+ private Mutable<ILogicalExpression> beforeOpFilterExpr;
private final Kind operation;
private final boolean bulkload;
private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
@@ -89,12 +90,13 @@
public IndexInsertDeleteUpsertOperator(IDataSourceIndex<?, ?> dataSourceIndex,
List<Mutable<ILogicalExpression>> primaryKeyExprs, List<Mutable<ILogicalExpression>> secondaryKeyExprs,
- Mutable<ILogicalExpression> filterExpr, Kind operation, boolean bulkload,
- int numberOfAdditionalNonFilteringFields) {
+ Mutable<ILogicalExpression> filterExpr, Mutable<ILogicalExpression> beforeOpFilterExpr, Kind operation,
+ boolean bulkload, int numberOfAdditionalNonFilteringFields) {
this.dataSourceIndex = dataSourceIndex;
this.primaryKeyExprs = primaryKeyExprs;
this.secondaryKeyExprs = secondaryKeyExprs;
this.filterExpr = filterExpr;
+ this.beforeOpFilterExpr = beforeOpFilterExpr;
this.operation = operation;
this.bulkload = bulkload;
this.numberOfAdditionalNonFilteringFields = numberOfAdditionalNonFilteringFields;
@@ -121,6 +123,12 @@
b = true;
}
}
+ // Old Filtering <For upsert>
+ if (beforeOpFilterExpr != null) {
+ if (visitor.transform(beforeOpFilterExpr)) {
+ b = true;
+ }
+ }
// Additional Filtering <For upsert>
if (additionalFilteringExpressions != null) {
for (int i = 0; i < additionalFilteringExpressions.size(); i++) {
@@ -141,7 +149,7 @@
}
}
}
- // Old Filtering <For upsert>
+ // Old Additional Filtering <For upsert>
if (prevAdditionalFilteringExpression != null) {
visitor.transform(prevAdditionalFilteringExpression);
}
@@ -164,6 +172,9 @@
if (getFilterExpression() != null) {
getFilterExpression().getValue().getUsedVariables(vars);
}
+ if (getBeforeOpFilterExpression() != null) {
+ getBeforeOpFilterExpression().getValue().getUsedVariables(vars);
+ }
if (getAdditionalFilteringExpressions() != null) {
for (Mutable<ILogicalExpression> e : getAdditionalFilteringExpressions()) {
e.getValue().getUsedVariables(vars);
@@ -232,10 +243,18 @@
return filterExpr;
}
+ public Mutable<ILogicalExpression> getBeforeOpFilterExpression() {
+ return beforeOpFilterExpr;
+ }
+
public void setFilterExpression(Mutable<ILogicalExpression> filterExpr) {
this.filterExpr = filterExpr;
}
+ public void setBeforeOpFilterExpression(Mutable<ILogicalExpression> beforeOpFilterExpr) {
+ this.beforeOpFilterExpr = beforeOpFilterExpr;
+ }
+
public Kind getOperation() {
return operation;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
index c4b4e47..aa5b5b4 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
@@ -610,6 +610,7 @@
|| !Objects.equals(op.getPrimaryKeyExpressions(), insertOpArg.getPrimaryKeyExpressions())
|| !Objects.equals(op.getSecondaryKeyExpressions(), insertOpArg.getSecondaryKeyExpressions())
|| !Objects.equals(op.getFilterExpression(), insertOpArg.getFilterExpression())
+ || !Objects.equals(op.getBeforeOpFilterExpression(), insertOpArg.getBeforeOpFilterExpression())
|| !Objects.equals(op.getOperation(), insertOpArg.getOperation())
|| (op.isBulkload() != insertOpArg.isBulkload())
|| !Objects.equals(op.getAdditionalFilteringExpressions(),
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java
index c06ad5c..1c91d7b 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java
@@ -324,11 +324,14 @@
deepCopyExpressionRefs(newSecondaryKeyExpressions, op.getSecondaryKeyExpressions());
Mutable<ILogicalExpression> newFilterExpression =
new MutableObject<>(((AbstractLogicalExpression) op.getFilterExpression()).cloneExpression());
+ Mutable<ILogicalExpression> newBeforeOpFilterExpression =
+ new MutableObject<>(((AbstractLogicalExpression) op.getBeforeOpFilterExpression()).cloneExpression());
List<Mutable<ILogicalExpression>> newLSMComponentFilterExpressions = new ArrayList<>();
deepCopyExpressionRefs(newLSMComponentFilterExpressions, op.getAdditionalFilteringExpressions());
- IndexInsertDeleteUpsertOperator indexInsertDeleteOp = new IndexInsertDeleteUpsertOperator(
- op.getDataSourceIndex(), newPrimaryKeyExpressions, newSecondaryKeyExpressions, newFilterExpression,
- op.getOperation(), op.isBulkload(), op.getNumberOfAdditionalNonFilteringFields());
+ IndexInsertDeleteUpsertOperator indexInsertDeleteOp =
+ new IndexInsertDeleteUpsertOperator(op.getDataSourceIndex(), newPrimaryKeyExpressions,
+ newSecondaryKeyExpressions, newFilterExpression, newBeforeOpFilterExpression, op.getOperation(),
+ op.isBulkload(), op.getNumberOfAdditionalNonFilteringFields());
indexInsertDeleteOp.setAdditionalFilteringExpressions(newLSMComponentFilterExpressions);
for (ILogicalPlan plan : op.getNestedPlans()) {
indexInsertDeleteOp.getNestedPlans().add(OperatorManipulationUtil.deepCopy(plan, indexInsertDeleteOp));
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
index 5aa63ae..439e493 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
@@ -452,6 +452,7 @@
substUsedVariablesInExpr(op.getPrimaryKeyExpressions(), pair.first, pair.second);
substUsedVariablesInExpr(op.getSecondaryKeyExpressions(), pair.first, pair.second);
substUsedVariablesInExpr(op.getFilterExpression(), pair.first, pair.second);
+ substUsedVariablesInExpr(op.getBeforeOpFilterExpression(), pair.first, pair.second);
substUsedVariablesInExpr(op.getAdditionalFilteringExpressions(), pair.first, pair.second);
substUsedVariablesInExpr(op.getOperationExpr(), pair.first, pair.second);
substUsedVariablesInExpr(op.getPrevSecondaryKeyExprs(), pair.first, pair.second);
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
index 4fb30b4..174b184 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
@@ -416,6 +416,9 @@
if (op.getFilterExpression() != null) {
op.getFilterExpression().getValue().getUsedVariables(usedVariables);
}
+ if (op.getBeforeOpFilterExpression() != null) {
+ op.getBeforeOpFilterExpression().getValue().getUsedVariables(usedVariables);
+ }
if (op.getAdditionalFilteringExpressions() != null) {
for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) {
e.getValue().getUsedVariables(usedVariables);
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeleteUpsertPOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeleteUpsertPOperator.java
index 26581b1..3416163 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeleteUpsertPOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeleteUpsertPOperator.java
@@ -55,6 +55,7 @@
private final List<LogicalVariable> primaryKeys;
private final List<LogicalVariable> secondaryKeys;
private final ILogicalExpression filterExpr;
+ private final ILogicalExpression prevFilterExpr;
private final IDataSourceIndex<?, ?> dataSourceIndex;
private final List<LogicalVariable> additionalFilteringKeys;
private final LogicalVariable operationVar;
@@ -64,9 +65,9 @@
public IndexInsertDeleteUpsertPOperator(List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
List<LogicalVariable> additionalFilteringKeys, Mutable<ILogicalExpression> filterExpr,
- IDataSourceIndex<?, ?> dataSourceIndex, LogicalVariable operationVar,
- List<LogicalVariable> prevSecondaryKeys, LogicalVariable prevAdditionalFilteringKey,
- int numOfAdditionalNonFilteringFields) {
+ Mutable<ILogicalExpression> prevFilterExpr, IDataSourceIndex<?, ?> dataSourceIndex,
+ LogicalVariable operationVar, List<LogicalVariable> prevSecondaryKeys,
+ LogicalVariable prevAdditionalFilteringKey, int numOfAdditionalNonFilteringFields) {
this.primaryKeys = primaryKeys;
this.secondaryKeys = secondaryKeys;
if (filterExpr != null) {
@@ -74,6 +75,11 @@
} else {
this.filterExpr = null;
}
+ if (prevFilterExpr != null) {
+ this.prevFilterExpr = prevFilterExpr.getValue();
+ } else {
+ this.prevFilterExpr = null;
+ }
this.dataSourceIndex = dataSourceIndex;
this.additionalFilteringKeys = additionalFilteringKeys;
this.operationVar = operationVar;
@@ -157,8 +163,9 @@
break;
case UPSERT:
runtimeAndConstraints = mp.getIndexUpsertRuntime(dataSourceIndex, propagatedSchema, inputSchemas,
- typeEnv, primaryKeys, secondaryKeys, additionalFilteringKeys, filterExpr, operationVar,
- prevSecondaryKeys, prevAdditionalFilteringKey, inputDesc, context, spec, secondaryKeyPipelines);
+ typeEnv, primaryKeys, secondaryKeys, additionalFilteringKeys, filterExpr, prevFilterExpr,
+ operationVar, prevSecondaryKeys, prevAdditionalFilteringKey, inputDesc, context, spec,
+ secondaryKeyPipelines);
break;
default:
throw new AlgebricksException("Unsupported Operation " + operation);
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
index cd0d996..a6fe495 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
@@ -442,8 +442,9 @@
}
}
return new IndexInsertDeleteUpsertPOperator(primaryKeys, secondaryKeys, additionalFilteringKeys,
- opInsDel.getFilterExpression(), opInsDel.getDataSourceIndex(), operationVar, prevSecondaryKeys,
- prevAdditionalFilteringKey, opInsDel.getNumberOfAdditionalNonFilteringFields());
+ opInsDel.getFilterExpression(), opInsDel.getBeforeOpFilterExpression(),
+ opInsDel.getDataSourceIndex(), operationVar, prevSecondaryKeys, prevAdditionalFilteringKey,
+ opInsDel.getNumberOfAdditionalNonFilteringFields());
}
}