Enabled inverted index insert and delete operations. Added Tests cases for all the 4 index types.
git-svn-id: https://asterixdb.googlecode.com/svn/branches/asterix_lsm_stabilization@1440 eaa15691-b419-025a-1212-ee371bd00084
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
index 6d21b27..49ca5ba 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
@@ -19,6 +19,7 @@
import edu.uci.ics.asterix.om.types.ARecordType;
import edu.uci.ics.asterix.om.types.ATypeTag;
import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
import edu.uci.ics.asterix.optimizer.rules.am.InvertedIndexAccessMethod;
import edu.uci.ics.asterix.optimizer.rules.am.InvertedIndexAccessMethod.SearchModifierType;
import edu.uci.ics.asterix.optimizer.rules.am.InvertedIndexJobGenParams;
@@ -159,9 +160,8 @@
ITypeTraits[] tokenTypeTraits = new ITypeTraits[numTokenKeys];
IBinaryComparatorFactory[] tokenComparatorFactories = new IBinaryComparatorFactory[numTokenKeys];
for (int i = 0; i < numSecondaryKeys; i++) {
- tokenComparatorFactories[i] = InvertedIndexAccessMethod
- .getTokenBinaryComparatorFactory(secondaryKeyType);
- tokenTypeTraits[i] = InvertedIndexAccessMethod.getTokenTypeTrait(secondaryKeyType);
+ tokenComparatorFactories[i] = NonTaggedFormatUtil.getTokenBinaryComparatorFactory(secondaryKeyType);
+ tokenTypeTraits[i] = NonTaggedFormatUtil.getTokenTypeTrait(secondaryKeyType);
}
if (isPartitioned) {
// The partitioning field is hardcoded to be a short *without* an Asterix type tag.
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index 3310f4d..fed591d 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -146,7 +146,10 @@
project.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
context.computeAndSetTypeEnvironmentForOperator(project);
context.computeAndSetTypeEnvironmentForOperator(assign);
- if (index.getIndexType() == IndexType.BTREE) {
+ if (index.getIndexType() == IndexType.BTREE || index.getIndexType() == IndexType.WORD_INVIX
+ || index.getIndexType() == IndexType.NGRAM_INVIX
+ || index.getIndexType() == IndexType.FUZZY_WORD_INVIX
+ || index.getIndexType() == IndexType.FUZZY_NGRAM_INVIX) {
for (LogicalVariable secondaryKeyVar : secondaryKeyVars) {
secondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
secondaryKeyVar)));
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
index d4d6b43..ff260f1 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
@@ -12,9 +12,7 @@
import edu.uci.ics.asterix.algebra.base.LogicalOperatorDeepCopyVisitor;
import edu.uci.ics.asterix.aql.util.FunctionUtils;
import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
-import edu.uci.ics.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
import edu.uci.ics.asterix.formats.nontagged.AqlBinaryTokenizerFactoryProvider;
-import edu.uci.ics.asterix.formats.nontagged.AqlTypeTraitProvider;
import edu.uci.ics.asterix.metadata.entities.Dataset;
import edu.uci.ics.asterix.metadata.entities.Index;
import edu.uci.ics.asterix.om.base.AFloat;
@@ -25,11 +23,8 @@
import edu.uci.ics.asterix.om.base.IAObject;
import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.asterix.om.types.AOrderedListType;
import edu.uci.ics.asterix.om.types.ARecordType;
import edu.uci.ics.asterix.om.types.ATypeTag;
-import edu.uci.ics.asterix.om.types.AUnorderedListType;
-import edu.uci.ics.asterix.om.types.AbstractCollectionType;
import edu.uci.ics.asterix.om.types.IAType;
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
@@ -57,8 +52,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifierFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.search.ConjunctiveSearchModifierFactory;
import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.search.EditDistanceSearchModifierFactory;
@@ -519,37 +512,37 @@
joinInputSubTreeVarMap.put(optFuncExpr.getLogicalVar(i), context.newVar());
newProbeSubTreeVarMap.put(optFuncExpr.getLogicalVar(i), optFuncExpr.getLogicalVar(i));
}
- for (int i = 0; i < originalSubTreePKs.size(); i++) {
+ for (int i = 0; i < originalSubTreePKs.size(); i++) {
LogicalVariable newPKVar = context.newVar();
surrogateSubTreePKs.add(newPKVar);
joinInputSubTreeVarMap.put(originalSubTreePKs.get(i), newPKVar);
newProbeSubTreeVarMap.put(originalSubTreePKs.get(i), originalSubTreePKs.get(i));
}
-
+
// Create first copy.
Counter firstCounter = new Counter(context.getVarCounter());
LogicalOperatorDeepCopyVisitor firstDeepCopyVisitor = new LogicalOperatorDeepCopyVisitor(firstCounter,
newProbeSubTreeVarMap);
ILogicalOperator newProbeSubTree = firstDeepCopyVisitor.deepCopy(probeSubTree.root, null);
inferTypes(newProbeSubTree, context);
- Mutable<ILogicalOperator> newProbeSubTreeRootRef = new MutableObject<ILogicalOperator>(newProbeSubTree);
+ Mutable<ILogicalOperator> newProbeSubTreeRootRef = new MutableObject<ILogicalOperator>(newProbeSubTree);
context.setVarCounter(firstCounter.get());
// Create second copy.
Counter secondCounter = new Counter(context.getVarCounter());
LogicalOperatorDeepCopyVisitor secondDeepCopyVisitor = new LogicalOperatorDeepCopyVisitor(secondCounter,
- joinInputSubTreeVarMap);
+ joinInputSubTreeVarMap);
ILogicalOperator joinInputSubTree = secondDeepCopyVisitor.deepCopy(probeSubTree.root, null);
inferTypes(joinInputSubTree, context);
probeSubTree.rootRef.setValue(joinInputSubTree);
context.setVarCounter(secondCounter.get());
-
+
// Remember the original probe subtree reference so we can return it.
Mutable<ILogicalOperator> originalProbeSubTreeRootRef = probeSubTree.rootRef;
// Replace the original probe subtree with its copy.
Dataset origDataset = probeSubTree.dataset;
ARecordType origRecordType = probeSubTree.recordType;
- probeSubTree.initFromSubTree(newProbeSubTreeRootRef);
+ probeSubTree.initFromSubTree(newProbeSubTreeRootRef);
probeSubTree.dataset = origDataset;
probeSubTree.recordType = origRecordType;
@@ -571,7 +564,7 @@
for (int i = 0; i < numPKVars; i++) {
List<Mutable<ILogicalExpression>> args = new ArrayList<Mutable<ILogicalExpression>>();
args.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(surrogateSubTreePKs.get(i))));
- args.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(originalSubTreePKs.get(i))));
+ args.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(originalSubTreePKs.get(i))));
ILogicalExpression eqFunc = new ScalarFunctionCallExpression(
FunctionUtils.getFunctionInfo(AlgebricksBuiltinFunctions.EQ), args);
eqExprs.add(new MutableObject<ILogicalExpression>(eqFunc));
@@ -839,42 +832,6 @@
return false;
}
- public static IBinaryComparatorFactory getTokenBinaryComparatorFactory(IAType keyType) throws AlgebricksException {
- IAType type = keyType;
- ATypeTag typeTag = keyType.getTypeTag();
- // Extract item type from list.
- if (typeTag == ATypeTag.UNORDEREDLIST || typeTag == ATypeTag.ORDEREDLIST) {
- AbstractCollectionType listType = (AbstractCollectionType) keyType;
- if (!listType.isTyped()) {
- throw new AlgebricksException("Cannot build an inverted index on untyped lists.)");
- }
- type = listType.getItemType();
- }
- // Ignore case for string types.
- return AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(type, true, true);
- }
-
- public static ITypeTraits getTokenTypeTrait(IAType keyType) throws AlgebricksException {
- IAType type = keyType;
- ATypeTag typeTag = keyType.getTypeTag();
- // Extract item type from list.
- if (typeTag == ATypeTag.UNORDEREDLIST) {
- AUnorderedListType ulistType = (AUnorderedListType) keyType;
- if (!ulistType.isTyped()) {
- throw new AlgebricksException("Cannot build an inverted index on untyped lists.)");
- }
- type = ulistType.getItemType();
- }
- if (typeTag == ATypeTag.ORDEREDLIST) {
- AOrderedListType olistType = (AOrderedListType) keyType;
- if (!olistType.isTyped()) {
- throw new AlgebricksException("Cannot build an inverted index on untyped lists.)");
- }
- type = olistType.getItemType();
- }
- return AqlTypeTraitProvider.INSTANCE.getTypeTrait(type);
- }
-
public static IBinaryTokenizerFactory getBinaryTokenizerFactory(SearchModifierType searchModifierType,
ATypeTag searchKeyType, Index index) throws AlgebricksException {
switch (index.getIndexType()) {
@@ -895,24 +852,6 @@
}
}
- public static IBinaryTokenizerFactory getBinaryTokenizerFactory(ATypeTag keyType, IndexType indexType,
- int gramLength) throws AlgebricksException {
- switch (indexType) {
- case WORD_INVIX:
- case FUZZY_WORD_INVIX: {
- return AqlBinaryTokenizerFactoryProvider.INSTANCE.getWordTokenizerFactory(keyType, false);
- }
- case NGRAM_INVIX:
- case FUZZY_NGRAM_INVIX: {
- return AqlBinaryTokenizerFactoryProvider.INSTANCE.getNGramTokenizerFactory(keyType, gramLength, true,
- false);
- }
- default: {
- throw new AlgebricksException("Tokenizer not applicable to index type '" + indexType + "'.");
- }
- }
- }
-
public static IInvertedIndexSearchModifierFactory getSearchModifierFactory(SearchModifierType searchModifierType,
IAObject simThresh, Index index) throws AlgebricksException {
switch (searchModifierType) {
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
index daa6d66..62a92cc 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
@@ -9,7 +9,7 @@
import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
import edu.uci.ics.asterix.metadata.entities.Index;
import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.asterix.optimizer.rules.am.InvertedIndexAccessMethod;
+import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
import edu.uci.ics.asterix.runtime.formats.FormatUtils;
import edu.uci.ics.asterix.transaction.management.resource.ILocalResourceMetadata;
import edu.uci.ics.asterix.transaction.management.resource.LSMInvertedIndexLocalResourceMetadata;
@@ -104,8 +104,8 @@
int numTokenFields = (!isPartitioned) ? numSecondaryKeys : numSecondaryKeys + 1;
tokenComparatorFactories = new IBinaryComparatorFactory[numTokenFields];
tokenTypeTraits = new ITypeTraits[numTokenFields];
- tokenComparatorFactories[0] = InvertedIndexAccessMethod.getTokenBinaryComparatorFactory(secondaryKeyType);
- tokenTypeTraits[0] = InvertedIndexAccessMethod.getTokenTypeTrait(secondaryKeyType);
+ tokenComparatorFactories[0] = NonTaggedFormatUtil.getTokenBinaryComparatorFactory(secondaryKeyType);
+ tokenTypeTraits[0] = NonTaggedFormatUtil.getTokenTypeTrait(secondaryKeyType);
if (isPartitioned) {
// The partitioning field is hardcoded to be a short *without* an Asterix type tag.
tokenComparatorFactories[1] = PointableBinaryComparatorFactory.of(ShortPointable.FACTORY);
@@ -114,7 +114,7 @@
// Set tokenizer factory.
// TODO: We might want to expose the hashing option at the AQL level,
// and add the choice to the index metadata.
- tokenizerFactory = InvertedIndexAccessMethod.getBinaryTokenizerFactory(secondaryKeyType.getTypeTag(),
+ tokenizerFactory = NonTaggedFormatUtil.getBinaryTokenizerFactory(secondaryKeyType.getTypeTag(),
createIndexStmt.getIndexType(), createIndexStmt.getGramLength());
// Type traits for inverted-list elements. Inverted lists contain
// primary keys.
@@ -130,8 +130,7 @@
tokenKeyPairComparatorFactories = new IBinaryComparatorFactory[numTokenKeyPairFields];
tokenKeyPairFields[0] = serdeProvider.getSerializerDeserializer(secondaryKeyType);
tokenKeyPairTypeTraits[0] = tokenTypeTraits[0];
- tokenKeyPairComparatorFactories[0] = InvertedIndexAccessMethod
- .getTokenBinaryComparatorFactory(secondaryKeyType);
+ tokenKeyPairComparatorFactories[0] = NonTaggedFormatUtil.getTokenBinaryComparatorFactory(secondaryKeyType);
int pkOff = 1;
if (isPartitioned) {
tokenKeyPairFields[1] = ShortSerializerDeserializer.INSTANCE;
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.1.ddl.aql
new file mode 100644
index 0000000..ec978fe
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.1.ddl.aql
@@ -0,0 +1,17 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string?,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.2.update.aql
new file mode 100644
index 0000000..a41d826
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-nulls.adm"),("format"="adm"));
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.3.ddl.aql
new file mode 100644
index 0000000..237d81c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.3.ddl.aql
@@ -0,0 +1,3 @@
+use dataverse test;
+
+create index fuzzy_ngram_index on DBLP(title) type fuzzy ngram(3);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.4.update.aql
new file mode 100644
index 0000000..2196231
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.4.update.aql
@@ -0,0 +1,3 @@
+use dataverse test;
+
+delete $o from dataset DBLP where $o.id>50;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.5.query.aql
new file mode 100644
index 0000000..866b045
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.5.query.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+for $o in dataset('DBLP')
+where contains($o.title, "Multimedia")
+order by $o.id
+return $o
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.1.ddl.aql
new file mode 100644
index 0000000..b4bfd7d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.1.ddl.aql
@@ -0,0 +1,16 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.2.update.aql
new file mode 100644
index 0000000..2446c57
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-id.txt"),("format"="delimited-text"),("delimiter"=":")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.3.ddl.aql
new file mode 100644
index 0000000..237d81c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.3.ddl.aql
@@ -0,0 +1,3 @@
+use dataverse test;
+
+create index fuzzy_ngram_index on DBLP(title) type fuzzy ngram(3);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.4.update.aql
new file mode 100644
index 0000000..2196231
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.4.update.aql
@@ -0,0 +1,3 @@
+use dataverse test;
+
+delete $o from dataset DBLP where $o.id>50;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.5.query.aql
new file mode 100644
index 0000000..866b045
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.5.query.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+for $o in dataset('DBLP')
+where contains($o.title, "Multimedia")
+order by $o.id
+return $o
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.1.ddl.aql
new file mode 100644
index 0000000..ec978fe
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.1.ddl.aql
@@ -0,0 +1,17 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string?,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.2.update.aql
new file mode 100644
index 0000000..a41d826
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-nulls.adm"),("format"="adm"));
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.3.ddl.aql
new file mode 100644
index 0000000..dec6c2e
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.3.ddl.aql
@@ -0,0 +1,3 @@
+use dataverse test;
+
+create index fuzzy_keyword_index on DBLP(title) type fuzzy keyword;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.4.update.aql
new file mode 100644
index 0000000..323f6c5
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.4.update.aql
@@ -0,0 +1,3 @@
+use dataverse test;
+
+delete $o from dataset DBLP where $o.id<50;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.5.query.aql
new file mode 100644
index 0000000..7ce908c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.5.query.aql
@@ -0,0 +1,7 @@
+use dataverse test;
+
+for $o in dataset('DBLP')
+let $jacc := similarity-jaccard-check(word-tokens($o.title), word-tokens("Transactions for Cooperative Environments"), 0.5f)
+where $jacc[0]
+return $o
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.1.ddl.aql
new file mode 100644
index 0000000..7833017
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.1.ddl.aql
@@ -0,0 +1,17 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.2.update.aql
new file mode 100644
index 0000000..2446c57
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-id.txt"),("format"="delimited-text"),("delimiter"=":")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.3.ddl.aql
new file mode 100644
index 0000000..dec6c2e
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.3.ddl.aql
@@ -0,0 +1,3 @@
+use dataverse test;
+
+create index fuzzy_keyword_index on DBLP(title) type fuzzy keyword;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.4.update.aql
new file mode 100644
index 0000000..323f6c5
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.4.update.aql
@@ -0,0 +1,3 @@
+use dataverse test;
+
+delete $o from dataset DBLP where $o.id<50;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.5.query.aql
new file mode 100644
index 0000000..7ce908c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.5.query.aql
@@ -0,0 +1,7 @@
+use dataverse test;
+
+for $o in dataset('DBLP')
+let $jacc := similarity-jaccard-check(word-tokens($o.title), word-tokens("Transactions for Cooperative Environments"), 0.5f)
+where $jacc[0]
+return $o
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.1.ddl.aql
new file mode 100644
index 0000000..ec978fe
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.1.ddl.aql
@@ -0,0 +1,17 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string?,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.2.update.aql
new file mode 100644
index 0000000..a41d826
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-nulls.adm"),("format"="adm"));
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.3.ddl.aql
new file mode 100644
index 0000000..420058b
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.3.ddl.aql
@@ -0,0 +1,3 @@
+use dataverse test;
+
+create index ngram_index on DBLP(title) type ngram(3);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.4.update.aql
new file mode 100644
index 0000000..2196231
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.4.update.aql
@@ -0,0 +1,3 @@
+use dataverse test;
+
+delete $o from dataset DBLP where $o.id>50;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.5.query.aql
new file mode 100644
index 0000000..866b045
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.5.query.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+for $o in dataset('DBLP')
+where contains($o.title, "Multimedia")
+order by $o.id
+return $o
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.1.ddl.aql
new file mode 100644
index 0000000..b4bfd7d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.1.ddl.aql
@@ -0,0 +1,16 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.2.update.aql
new file mode 100644
index 0000000..2446c57
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-id.txt"),("format"="delimited-text"),("delimiter"=":")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.3.ddl.aql
new file mode 100644
index 0000000..420058b
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.3.ddl.aql
@@ -0,0 +1,3 @@
+use dataverse test;
+
+create index ngram_index on DBLP(title) type ngram(3);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.4.update.aql
new file mode 100644
index 0000000..2196231
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.4.update.aql
@@ -0,0 +1,3 @@
+use dataverse test;
+
+delete $o from dataset DBLP where $o.id>50;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.5.query.aql
new file mode 100644
index 0000000..866b045
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.5.query.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+for $o in dataset('DBLP')
+where contains($o.title, "Multimedia")
+order by $o.id
+return $o
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.1.ddl.aql
new file mode 100644
index 0000000..c5655c9
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.1.ddl.aql
@@ -0,0 +1,15 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string?,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.2.update.aql
new file mode 100644
index 0000000..a41d826
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-nulls.adm"),("format"="adm"));
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.3.ddl.aql
new file mode 100644
index 0000000..59f9c68
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.3.ddl.aql
@@ -0,0 +1,3 @@
+use dataverse test;
+
+create index keyword_index on DBLP(title) type keyword;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.4.update.aql
new file mode 100644
index 0000000..323f6c5
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.4.update.aql
@@ -0,0 +1,3 @@
+use dataverse test;
+
+delete $o from dataset DBLP where $o.id<50;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.5.query.aql
new file mode 100644
index 0000000..7ce908c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.5.query.aql
@@ -0,0 +1,7 @@
+use dataverse test;
+
+for $o in dataset('DBLP')
+let $jacc := similarity-jaccard-check(word-tokens($o.title), word-tokens("Transactions for Cooperative Environments"), 0.5f)
+where $jacc[0]
+return $o
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.1.ddl.aql
new file mode 100644
index 0000000..b4bfd7d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.1.ddl.aql
@@ -0,0 +1,16 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.2.update.aql
new file mode 100644
index 0000000..2446c57
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-id.txt"),("format"="delimited-text"),("delimiter"=":")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.3.ddl.aql
new file mode 100644
index 0000000..59f9c68
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.3.ddl.aql
@@ -0,0 +1,3 @@
+use dataverse test;
+
+create index keyword_index on DBLP(title) type keyword;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.4.update.aql
new file mode 100644
index 0000000..323f6c5
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.4.update.aql
@@ -0,0 +1,3 @@
+use dataverse test;
+
+delete $o from dataset DBLP where $o.id<50;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.5.query.aql
new file mode 100644
index 0000000..7ce908c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.5.query.aql
@@ -0,0 +1,7 @@
+use dataverse test;
+
+for $o in dataset('DBLP')
+let $jacc := similarity-jaccard-check(word-tokens($o.title), word-tokens("Transactions for Cooperative Environments"), 0.5f)
+where $jacc[0]
+return $o
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.1.ddl.aql
new file mode 100644
index 0000000..7197873
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.1.ddl.aql
@@ -0,0 +1,18 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string?,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+create dataset DBLP1(DBLPType) primary key id;
+
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.2.update.aql
new file mode 100644
index 0000000..a41d826
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-nulls.adm"),("format"="adm"));
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.3.ddl.aql
new file mode 100644
index 0000000..94006b5
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.3.ddl.aql
@@ -0,0 +1,4 @@
+use dataverse test;
+
+create index fuzzy_ngram_index on DBLP(title) type fuzzy ngram(3);
+create index fuzzy_ngram_index1 on DBLP1(title) type fuzzy ngram(3);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.4.update.aql
new file mode 100644
index 0000000..b4e5a8b
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.4.update.aql
@@ -0,0 +1,14 @@
+use dataverse test;
+
+insert into dataset DBLP1 (
+for $o in dataset('DBLP')
+where contains($o.title, "Multimedia")
+order by $o.id
+return {
+ "id": $o.id,
+ "dblpid": $o.dblpid,
+ "title": $o.title,
+ "authors": $o.authors,
+ "misc": $o.misc
+ }
+);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.5.query.aql
new file mode 100644
index 0000000..7cb7484
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.5.query.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+for $o in dataset('DBLP1')
+where contains($o.title, "Multimedia")
+order by $o.id
+return $o
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.1.ddl.aql
new file mode 100644
index 0000000..770dba5
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.1.ddl.aql
@@ -0,0 +1,18 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+create dataset DBLP1(DBLPType) primary key id;
+
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.2.update.aql
new file mode 100644
index 0000000..2446c57
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-id.txt"),("format"="delimited-text"),("delimiter"=":")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.3.ddl.aql
new file mode 100644
index 0000000..94006b5
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.3.ddl.aql
@@ -0,0 +1,4 @@
+use dataverse test;
+
+create index fuzzy_ngram_index on DBLP(title) type fuzzy ngram(3);
+create index fuzzy_ngram_index1 on DBLP1(title) type fuzzy ngram(3);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.4.update.aql
new file mode 100644
index 0000000..b4e5a8b
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.4.update.aql
@@ -0,0 +1,14 @@
+use dataverse test;
+
+insert into dataset DBLP1 (
+for $o in dataset('DBLP')
+where contains($o.title, "Multimedia")
+order by $o.id
+return {
+ "id": $o.id,
+ "dblpid": $o.dblpid,
+ "title": $o.title,
+ "authors": $o.authors,
+ "misc": $o.misc
+ }
+);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.5.query.aql
new file mode 100644
index 0000000..7cb7484
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.5.query.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+for $o in dataset('DBLP1')
+where contains($o.title, "Multimedia")
+order by $o.id
+return $o
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.1.ddl.aql
new file mode 100644
index 0000000..7197873
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.1.ddl.aql
@@ -0,0 +1,18 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string?,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+create dataset DBLP1(DBLPType) primary key id;
+
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.2.update.aql
new file mode 100644
index 0000000..a41d826
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-nulls.adm"),("format"="adm"));
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.3.ddl.aql
new file mode 100644
index 0000000..0b54e33
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.3.ddl.aql
@@ -0,0 +1,4 @@
+use dataverse test;
+
+create index fuzzy_keyword_index on DBLP(title) type fuzzy keyword;
+create index fuzzy_keyword_index1 on DBLP1(title) type fuzzy keyword;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.4.update.aql
new file mode 100644
index 0000000..4fb7d93
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.4.update.aql
@@ -0,0 +1,13 @@
+use dataverse test;
+
+insert into dataset DBLP1 (
+for $o in dataset('DBLP')
+order by $o.id
+return {
+ "id": $o.id,
+ "dblpid": $o.dblpid,
+ "title": $o.title,
+ "authors": $o.authors,
+ "misc": $o.misc
+ }
+);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.5.query.aql
new file mode 100644
index 0000000..0b2f3f9
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.5.query.aql
@@ -0,0 +1,7 @@
+use dataverse test;
+
+for $o in dataset('DBLP1')
+let $jacc := similarity-jaccard-check(word-tokens($o.title), word-tokens("Transactions for Cooperative Environments"), 0.5f)
+where $jacc[0]
+return $o
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.1.ddl.aql
new file mode 100644
index 0000000..770dba5
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.1.ddl.aql
@@ -0,0 +1,18 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+create dataset DBLP1(DBLPType) primary key id;
+
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.2.update.aql
new file mode 100644
index 0000000..2446c57
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-id.txt"),("format"="delimited-text"),("delimiter"=":")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.3.ddl.aql
new file mode 100644
index 0000000..0b54e33
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.3.ddl.aql
@@ -0,0 +1,4 @@
+use dataverse test;
+
+create index fuzzy_keyword_index on DBLP(title) type fuzzy keyword;
+create index fuzzy_keyword_index1 on DBLP1(title) type fuzzy keyword;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.4.update.aql
new file mode 100644
index 0000000..4fb7d93
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.4.update.aql
@@ -0,0 +1,13 @@
+use dataverse test;
+
+insert into dataset DBLP1 (
+for $o in dataset('DBLP')
+order by $o.id
+return {
+ "id": $o.id,
+ "dblpid": $o.dblpid,
+ "title": $o.title,
+ "authors": $o.authors,
+ "misc": $o.misc
+ }
+);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.5.query.aql
new file mode 100644
index 0000000..0b2f3f9
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.5.query.aql
@@ -0,0 +1,7 @@
+use dataverse test;
+
+for $o in dataset('DBLP1')
+let $jacc := similarity-jaccard-check(word-tokens($o.title), word-tokens("Transactions for Cooperative Environments"), 0.5f)
+where $jacc[0]
+return $o
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.1.ddl.aql
new file mode 100644
index 0000000..7197873
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.1.ddl.aql
@@ -0,0 +1,18 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string?,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+create dataset DBLP1(DBLPType) primary key id;
+
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.2.update.aql
new file mode 100644
index 0000000..a41d826
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-nulls.adm"),("format"="adm"));
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.3.ddl.aql
new file mode 100644
index 0000000..18b5027
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.3.ddl.aql
@@ -0,0 +1,4 @@
+use dataverse test;
+
+create index ngram_index on DBLP(title) type ngram(3);
+create index ngram_index1 on DBLP1(title) type ngram(3);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.4.update.aql
new file mode 100644
index 0000000..b4e5a8b
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.4.update.aql
@@ -0,0 +1,14 @@
+use dataverse test;
+
+insert into dataset DBLP1 (
+for $o in dataset('DBLP')
+where contains($o.title, "Multimedia")
+order by $o.id
+return {
+ "id": $o.id,
+ "dblpid": $o.dblpid,
+ "title": $o.title,
+ "authors": $o.authors,
+ "misc": $o.misc
+ }
+);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.5.query.aql
new file mode 100644
index 0000000..7cb7484
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.5.query.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+for $o in dataset('DBLP1')
+where contains($o.title, "Multimedia")
+order by $o.id
+return $o
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.1.ddl.aql
new file mode 100644
index 0000000..770dba5
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.1.ddl.aql
@@ -0,0 +1,18 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+create dataset DBLP1(DBLPType) primary key id;
+
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.2.update.aql
new file mode 100644
index 0000000..2446c57
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-id.txt"),("format"="delimited-text"),("delimiter"=":")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.3.ddl.aql
new file mode 100644
index 0000000..18b5027
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.3.ddl.aql
@@ -0,0 +1,4 @@
+use dataverse test;
+
+create index ngram_index on DBLP(title) type ngram(3);
+create index ngram_index1 on DBLP1(title) type ngram(3);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.4.update.aql
new file mode 100644
index 0000000..b4e5a8b
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.4.update.aql
@@ -0,0 +1,14 @@
+use dataverse test;
+
+insert into dataset DBLP1 (
+for $o in dataset('DBLP')
+where contains($o.title, "Multimedia")
+order by $o.id
+return {
+ "id": $o.id,
+ "dblpid": $o.dblpid,
+ "title": $o.title,
+ "authors": $o.authors,
+ "misc": $o.misc
+ }
+);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.5.query.aql
new file mode 100644
index 0000000..7cb7484
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.5.query.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+for $o in dataset('DBLP1')
+where contains($o.title, "Multimedia")
+order by $o.id
+return $o
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.1.ddl.aql
new file mode 100644
index 0000000..7197873
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.1.ddl.aql
@@ -0,0 +1,18 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string?,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+create dataset DBLP1(DBLPType) primary key id;
+
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.2.update.aql
new file mode 100644
index 0000000..a41d826
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-nulls.adm"),("format"="adm"));
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.3.ddl.aql
new file mode 100644
index 0000000..a1b7b6d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.3.ddl.aql
@@ -0,0 +1,4 @@
+use dataverse test;
+
+create index keyword_index on DBLP(title) type keyword;
+create index keyword_index1 on DBLP1(title) type keyword;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.4.update.aql
new file mode 100644
index 0000000..4fb7d93
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.4.update.aql
@@ -0,0 +1,13 @@
+use dataverse test;
+
+insert into dataset DBLP1 (
+for $o in dataset('DBLP')
+order by $o.id
+return {
+ "id": $o.id,
+ "dblpid": $o.dblpid,
+ "title": $o.title,
+ "authors": $o.authors,
+ "misc": $o.misc
+ }
+);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.5.query.aql
new file mode 100644
index 0000000..0b2f3f9
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.5.query.aql
@@ -0,0 +1,7 @@
+use dataverse test;
+
+for $o in dataset('DBLP1')
+let $jacc := similarity-jaccard-check(word-tokens($o.title), word-tokens("Transactions for Cooperative Environments"), 0.5f)
+where $jacc[0]
+return $o
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.1.ddl.aql
new file mode 100644
index 0000000..770dba5
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.1.ddl.aql
@@ -0,0 +1,18 @@
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int32,
+ dblpid: string,
+ title: string,
+ authors: string,
+ misc: string
+}
+
+create dataset DBLP(DBLPType) primary key id;
+create dataset DBLP1(DBLPType) primary key id;
+
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.2.update.aql
new file mode 100644
index 0000000..2446c57
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse test;
+
+load dataset DBLP using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/dblp-small/dblp-small-id.txt"),("format"="delimited-text"),("delimiter"=":")) pre-sorted;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.3.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.3.ddl.aql
new file mode 100644
index 0000000..a1b7b6d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.3.ddl.aql
@@ -0,0 +1,4 @@
+use dataverse test;
+
+create index keyword_index on DBLP(title) type keyword;
+create index keyword_index1 on DBLP1(title) type keyword;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.4.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.4.update.aql
new file mode 100644
index 0000000..4fb7d93
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.4.update.aql
@@ -0,0 +1,13 @@
+use dataverse test;
+
+insert into dataset DBLP1 (
+for $o in dataset('DBLP')
+order by $o.id
+return {
+ "id": $o.id,
+ "dblpid": $o.dblpid,
+ "title": $o.title,
+ "authors": $o.authors,
+ "misc": $o.misc
+ }
+);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.5.query.aql
new file mode 100644
index 0000000..0b2f3f9
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.5.query.aql
@@ -0,0 +1,7 @@
+use dataverse test;
+
+for $o in dataset('DBLP1')
+let $jacc := similarity-jaccard-check(word-tokens($o.title), word-tokens("Transactions for Cooperative Environments"), 0.5f)
+where $jacc[0]
+return $o
+
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.1.adm
new file mode 100644
index 0000000..7810e91
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-delete-inverted-index-fuzzy-ngram-secondary-index-nullable.1.adm
@@ -0,0 +1 @@
+{ "id": 4, "dblpid": "books/acm/kim95/ChristodoulakisK95", "title": "Multimedia Information Systems Issues and Approaches.", "authors": "Stavros Christodoulakis Leonidas Koveos", "misc": "2002-01-03 318-337 1995 Modern Database Systems db/books/collections/kim95.html#ChristodoulakisK95" }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.1.adm
new file mode 100644
index 0000000..7810e91
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-fuzzy-ngram-secondary-index/scan-delete-inverted-index-fuzzy-ngram-secondary-index.1.adm
@@ -0,0 +1 @@
+{ "id": 4, "dblpid": "books/acm/kim95/ChristodoulakisK95", "title": "Multimedia Information Systems Issues and Approaches.", "authors": "Stavros Christodoulakis Leonidas Koveos", "misc": "2002-01-03 318-337 1995 Modern Database Systems db/books/collections/kim95.html#ChristodoulakisK95" }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.1.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable/scan-delete-inverted-index-fuzzy-word-secondary-index-nullable.1.adm
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.1.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-fuzzy-word-secondary-index/scan-delete-inverted-index-fuzzy-word-secondary-index.1.adm
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.1.adm
new file mode 100644
index 0000000..7810e91
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-ngram-secondary-index-nullable/scan-delete-inverted-index-ngram-secondary-index-nullable.1.adm
@@ -0,0 +1 @@
+{ "id": 4, "dblpid": "books/acm/kim95/ChristodoulakisK95", "title": "Multimedia Information Systems Issues and Approaches.", "authors": "Stavros Christodoulakis Leonidas Koveos", "misc": "2002-01-03 318-337 1995 Modern Database Systems db/books/collections/kim95.html#ChristodoulakisK95" }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.1.adm
new file mode 100644
index 0000000..7810e91
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-ngram-secondary-index/scan-delete-inverted-index-ngram-secondary-index.1.adm
@@ -0,0 +1 @@
+{ "id": 4, "dblpid": "books/acm/kim95/ChristodoulakisK95", "title": "Multimedia Information Systems Issues and Approaches.", "authors": "Stavros Christodoulakis Leonidas Koveos", "misc": "2002-01-03 318-337 1995 Modern Database Systems db/books/collections/kim95.html#ChristodoulakisK95" }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.1.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-word-secondary-index-nullable/scan-delete-inverted-index-word-secondary-index-nullable.1.adm
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.1.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/scan-delete-inverted-index-word-secondary-index/scan-delete-inverted-index-word-secondary-index.1.adm
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.1.adm
new file mode 100644
index 0000000..8a99b26
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable/scan-insert-inverted-index-fuzzy-ngram-secondary-index-nullable.1.adm
@@ -0,0 +1,3 @@
+{ "id": 4, "dblpid": "books/acm/kim95/ChristodoulakisK95", "title": "Multimedia Information Systems Issues and Approaches.", "authors": "Stavros Christodoulakis Leonidas Koveos", "misc": "2002-01-03 318-337 1995 Modern Database Systems db/books/collections/kim95.html#ChristodoulakisK95" }
+{ "id": 89, "dblpid": "conf/icip/SchonfeldL98", "title": "VORTEX Video Retrieval and Tracking from Compressed Multimedia Databases.", "authors": "Dan Schonfeld Dan Lelescu", "misc": "2002-11-05 123-127 1998 ICIP (3) db/conf/icip/icip1998-3.html#SchonfeldL98" }
+{ "id": 90, "dblpid": "conf/hicss/SchonfeldL99", "title": "VORTEX Video Retrieval and Tracking from Compressed Multimedia Databases ¾ Visual Search Engine.", "authors": "Dan Schonfeld Dan Lelescu", "misc": "2002-01-03 1999 HICSS http //computer.org/proceedings/hicss/0001/00013/00013006abs.htm db/conf/hicss/hicss1999-3.html#SchonfeldL99" }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.1.adm
new file mode 100644
index 0000000..8a99b26
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-fuzzy-ngram-secondary-index/scan-insert-inverted-index-fuzzy-ngram-secondary-index.1.adm
@@ -0,0 +1,3 @@
+{ "id": 4, "dblpid": "books/acm/kim95/ChristodoulakisK95", "title": "Multimedia Information Systems Issues and Approaches.", "authors": "Stavros Christodoulakis Leonidas Koveos", "misc": "2002-01-03 318-337 1995 Modern Database Systems db/books/collections/kim95.html#ChristodoulakisK95" }
+{ "id": 89, "dblpid": "conf/icip/SchonfeldL98", "title": "VORTEX Video Retrieval and Tracking from Compressed Multimedia Databases.", "authors": "Dan Schonfeld Dan Lelescu", "misc": "2002-11-05 123-127 1998 ICIP (3) db/conf/icip/icip1998-3.html#SchonfeldL98" }
+{ "id": 90, "dblpid": "conf/hicss/SchonfeldL99", "title": "VORTEX Video Retrieval and Tracking from Compressed Multimedia Databases ¾ Visual Search Engine.", "authors": "Dan Schonfeld Dan Lelescu", "misc": "2002-01-03 1999 HICSS http //computer.org/proceedings/hicss/0001/00013/00013006abs.htm db/conf/hicss/hicss1999-3.html#SchonfeldL99" }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.1.adm
new file mode 100644
index 0000000..5bf6ae0
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable/scan-insert-inverted-index-fuzzy-word-secondary-index-nullable.1.adm
@@ -0,0 +1 @@
+{ "id": 9, "dblpid": "books/acm/kim95/Kaiser95", "title": "Cooperative Transactions for Multiuser Environments.", "authors": "Gail E. Kaiser", "misc": "2002-01-03 409-433 1995 Modern Database Systems db/books/collections/kim95.html#Kaiser95" }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.1.adm
new file mode 100644
index 0000000..5bf6ae0
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-fuzzy-word-secondary-index/scan-insert-inverted-index-fuzzy-word-secondary-index.1.adm
@@ -0,0 +1 @@
+{ "id": 9, "dblpid": "books/acm/kim95/Kaiser95", "title": "Cooperative Transactions for Multiuser Environments.", "authors": "Gail E. Kaiser", "misc": "2002-01-03 409-433 1995 Modern Database Systems db/books/collections/kim95.html#Kaiser95" }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.1.adm
new file mode 100644
index 0000000..8a99b26
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-ngram-secondary-index-nullable/scan-insert-inverted-index-ngram-secondary-index-nullable.1.adm
@@ -0,0 +1,3 @@
+{ "id": 4, "dblpid": "books/acm/kim95/ChristodoulakisK95", "title": "Multimedia Information Systems Issues and Approaches.", "authors": "Stavros Christodoulakis Leonidas Koveos", "misc": "2002-01-03 318-337 1995 Modern Database Systems db/books/collections/kim95.html#ChristodoulakisK95" }
+{ "id": 89, "dblpid": "conf/icip/SchonfeldL98", "title": "VORTEX Video Retrieval and Tracking from Compressed Multimedia Databases.", "authors": "Dan Schonfeld Dan Lelescu", "misc": "2002-11-05 123-127 1998 ICIP (3) db/conf/icip/icip1998-3.html#SchonfeldL98" }
+{ "id": 90, "dblpid": "conf/hicss/SchonfeldL99", "title": "VORTEX Video Retrieval and Tracking from Compressed Multimedia Databases ¾ Visual Search Engine.", "authors": "Dan Schonfeld Dan Lelescu", "misc": "2002-01-03 1999 HICSS http //computer.org/proceedings/hicss/0001/00013/00013006abs.htm db/conf/hicss/hicss1999-3.html#SchonfeldL99" }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.1.adm
new file mode 100644
index 0000000..8a99b26
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-ngram-secondary-index/scan-insert-inverted-index-ngram-secondary-index.1.adm
@@ -0,0 +1,3 @@
+{ "id": 4, "dblpid": "books/acm/kim95/ChristodoulakisK95", "title": "Multimedia Information Systems Issues and Approaches.", "authors": "Stavros Christodoulakis Leonidas Koveos", "misc": "2002-01-03 318-337 1995 Modern Database Systems db/books/collections/kim95.html#ChristodoulakisK95" }
+{ "id": 89, "dblpid": "conf/icip/SchonfeldL98", "title": "VORTEX Video Retrieval and Tracking from Compressed Multimedia Databases.", "authors": "Dan Schonfeld Dan Lelescu", "misc": "2002-11-05 123-127 1998 ICIP (3) db/conf/icip/icip1998-3.html#SchonfeldL98" }
+{ "id": 90, "dblpid": "conf/hicss/SchonfeldL99", "title": "VORTEX Video Retrieval and Tracking from Compressed Multimedia Databases ¾ Visual Search Engine.", "authors": "Dan Schonfeld Dan Lelescu", "misc": "2002-01-03 1999 HICSS http //computer.org/proceedings/hicss/0001/00013/00013006abs.htm db/conf/hicss/hicss1999-3.html#SchonfeldL99" }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.1.adm
new file mode 100644
index 0000000..5bf6ae0
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-word-secondary-index-nullable/scan-insert-inverted-index-word-secondary-index-nullable.1.adm
@@ -0,0 +1 @@
+{ "id": 9, "dblpid": "books/acm/kim95/Kaiser95", "title": "Cooperative Transactions for Multiuser Environments.", "authors": "Gail E. Kaiser", "misc": "2002-01-03 409-433 1995 Modern Database Systems db/books/collections/kim95.html#Kaiser95" }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.1.adm
new file mode 100644
index 0000000..5bf6ae0
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/scan-insert-inverted-index-word-secondary-index/scan-insert-inverted-index-word-secondary-index.1.adm
@@ -0,0 +1 @@
+{ "id": 9, "dblpid": "books/acm/kim95/Kaiser95", "title": "Cooperative Transactions for Multiuser Environments.", "authors": "Gail E. Kaiser", "misc": "2002-01-03 409-433 1995 Modern Database Systems db/books/collections/kim95.html#Kaiser95" }
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
index 1bba863..938dfc4 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
@@ -24,6 +24,7 @@
import java.util.logging.Logger;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
import edu.uci.ics.asterix.common.config.GlobalConfig;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.context.TransactionSubsystemProvider;
@@ -106,6 +107,8 @@
import edu.uci.ics.hyracks.api.dataset.ResultSetId;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.ShortPointable;
import edu.uci.ics.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
@@ -123,6 +126,9 @@
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexInsertUpdateDeleteOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexInsertUpdateDeleteOperator;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.dataflow.LSMRTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.rtree.dataflow.RTreeSearchOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
@@ -888,6 +894,14 @@
return getRTreeDmlRuntime(dataverseName, datasetName, indexName, propagatedSchema, typeEnv,
primaryKeys, secondaryKeys, filterFactory, recordDesc, context, spec, indexOp);
}
+ case WORD_INVIX:
+ case NGRAM_INVIX:
+ case FUZZY_WORD_INVIX:
+ case FUZZY_NGRAM_INVIX: {
+ return getInvertedIndexDmlRuntime(dataverseName, datasetName, indexName, propagatedSchema, typeEnv,
+ primaryKeys, secondaryKeys, filterFactory, recordDesc, context, spec, indexOp,
+ secondaryIndex.getIndexType());
+ }
default: {
throw new AlgebricksException("Insert and delete not implemented for index type: "
+ secondaryIndex.getIndexType());
@@ -1028,6 +1042,132 @@
}
}
+ private Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInvertedIndexDmlRuntime(String dataverseName,
+ String datasetName, String indexName, IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv,
+ List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
+ AsterixTupleFilterFactory filterFactory, RecordDescriptor recordDesc, JobGenContext context,
+ JobSpecification spec, IndexOperation indexOp, IndexType indexType) throws AlgebricksException {
+
+ // Sanity checks.
+ if (primaryKeys.size() > 1) {
+ throw new AlgebricksException("Cannot create inverted index on dataset with composite primary key.");
+ }
+ if (secondaryKeys.size() > 1) {
+ throw new AlgebricksException("Cannot create composite inverted index on multiple fields.");
+ }
+
+ int numKeys = primaryKeys.size() + secondaryKeys.size();
+ // generate field permutations
+ int[] fieldPermutation = new int[numKeys];
+ int i = 0;
+ for (LogicalVariable varKey : secondaryKeys) {
+ int idx = propagatedSchema.findVariable(varKey);
+ fieldPermutation[i] = idx;
+ i++;
+ }
+ for (LogicalVariable varKey : primaryKeys) {
+ int idx = propagatedSchema.findVariable(varKey);
+ fieldPermutation[i] = idx;
+ i++;
+ }
+
+ boolean isPartitioned;
+ if (indexType == IndexType.FUZZY_WORD_INVIX || indexType == IndexType.FUZZY_NGRAM_INVIX) {
+ isPartitioned = true;
+ } else {
+ isPartitioned = false;
+ }
+
+ Dataset dataset = findDataset(dataverseName, datasetName);
+ if (dataset == null) {
+ throw new AlgebricksException("Unknown dataset " + datasetName + " in dataverse " + dataverseName);
+ }
+ String itemTypeName = dataset.getItemTypeName();
+ IAType itemType;
+ try {
+ itemType = MetadataManager.INSTANCE.getDatatype(mdTxnCtx, dataset.getDataverseName(), itemTypeName)
+ .getDatatype();
+
+ if (itemType.getTypeTag() != ATypeTag.RECORD) {
+ throw new AlgebricksException("Only record types can be indexed.");
+ }
+
+ ARecordType recType = (ARecordType) itemType;
+
+ // Index parameters.
+ Index secondaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(),
+ dataset.getDatasetName(), indexName);
+
+ List<String> secondaryKeyExprs = secondaryIndex.getKeyFieldNames();
+
+ int numTokenFields = (!isPartitioned) ? secondaryKeys.size() : secondaryKeys.size() + 1;
+ ITypeTraits[] tokenTypeTraits = new ITypeTraits[numTokenFields];
+ ITypeTraits[] invListsTypeTraits = new ITypeTraits[primaryKeys.size()];
+ IBinaryComparatorFactory[] tokenComparatorFactories = new IBinaryComparatorFactory[numTokenFields];
+ IBinaryComparatorFactory[] invListComparatorFactories = new IBinaryComparatorFactory[primaryKeys.size()];
+
+ IAType secondaryKeyType = null;
+ for (i = 0; i < secondaryKeys.size(); ++i) {
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableKeyFieldType(secondaryKeyExprs.get(i)
+ .toString(), recType);
+ secondaryKeyType = keyPairType.first;
+ }
+ List<String> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
+ i = 0;
+ for (String partitioningKey : partitioningKeys) {
+ IAType keyType = recType.getFieldType(partitioningKey);
+ invListsTypeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(keyType);
+ ++i;
+ }
+
+ tokenComparatorFactories[0] = NonTaggedFormatUtil.getTokenBinaryComparatorFactory(secondaryKeyType);
+ tokenTypeTraits[0] = NonTaggedFormatUtil.getTokenTypeTrait(secondaryKeyType);
+ if (isPartitioned) {
+ // The partitioning field is hardcoded to be a short *without* an Asterix type tag.
+ tokenComparatorFactories[1] = PointableBinaryComparatorFactory.of(ShortPointable.FACTORY);
+ tokenTypeTraits[1] = ShortPointable.TYPE_TRAITS;
+ }
+ IBinaryTokenizerFactory tokenizerFactory = NonTaggedFormatUtil.getBinaryTokenizerFactory(
+ secondaryKeyType.getTypeTag(), indexType, secondaryIndex.getGramLength());
+
+ IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
+ Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = splitProviderAndPartitionConstraintsForInternalOrFeedDataset(
+ dataverseName, datasetName, indexName);
+
+ //prepare callback
+ JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
+ int datasetId = dataset.getDatasetId();
+ int[] primaryKeyFields = new int[primaryKeys.size()];
+ i = 0;
+ for (LogicalVariable varKey : primaryKeys) {
+ int idx = propagatedSchema.findVariable(varKey);
+ primaryKeyFields[i] = idx;
+ i++;
+ }
+ TransactionSubsystemProvider txnSubsystemProvider = new TransactionSubsystemProvider();
+ SecondaryIndexModificationOperationCallbackFactory modificationCallbackFactory = new SecondaryIndexModificationOperationCallbackFactory(
+ jobId, datasetId, primaryKeyFields, txnSubsystemProvider, indexOp, ResourceType.LSM_INVERTED_INDEX);
+
+ LSMInvertedIndexInsertUpdateDeleteOperator insertDeleteOp = new LSMInvertedIndexInsertUpdateDeleteOperator(
+ spec, recordDesc, appContext.getStorageManagerInterface(), splitsAndConstraint.first,
+ appContext.getIndexLifecycleManagerProvider(), tokenTypeTraits, tokenComparatorFactories,
+ invListsTypeTraits, invListComparatorFactories, tokenizerFactory, fieldPermutation, indexOp,
+ new LSMInvertedIndexDataflowHelperFactory(
+ AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
+ AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
+ AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
+ AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
+ GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
+ filterFactory, modificationCallbackFactory);
+ return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(insertDeleteOp,
+ splitsAndConstraint.second);
+ } catch (MetadataException e) {
+ throw new AlgebricksException(e);
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
private Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getRTreeDmlRuntime(String dataverseName,
String datasetName, String indexName, IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv,
List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/NonTaggedFormatUtil.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/NonTaggedFormatUtil.java
index a8c0485..047c459 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/NonTaggedFormatUtil.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/NonTaggedFormatUtil.java
@@ -2,18 +2,29 @@
import java.util.List;
+import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.dataflow.data.nontagged.serde.AInt16SerializerDeserializer;
import edu.uci.ics.asterix.dataflow.data.nontagged.serde.AOrderedListSerializerDeserializer;
import edu.uci.ics.asterix.dataflow.data.nontagged.serde.ARecordSerializerDeserializer;
import edu.uci.ics.asterix.dataflow.data.nontagged.serde.AUnorderedListSerializerDeserializer;
+import edu.uci.ics.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
+import edu.uci.ics.asterix.formats.nontagged.AqlBinaryTokenizerFactoryProvider;
+import edu.uci.ics.asterix.formats.nontagged.AqlTypeTraitProvider;
+import edu.uci.ics.asterix.om.types.AOrderedListType;
import edu.uci.ics.asterix.om.types.ARecordType;
import edu.uci.ics.asterix.om.types.ATypeTag;
import edu.uci.ics.asterix.om.types.AUnionType;
+import edu.uci.ics.asterix.om.types.AUnorderedListType;
+import edu.uci.ics.asterix.om.types.AbstractCollectionType;
import edu.uci.ics.asterix.om.types.BuiltinType;
import edu.uci.ics.asterix.om.types.EnumDeserializer;
import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
public final class NonTaggedFormatUtil {
@@ -160,4 +171,58 @@
throw new NotImplementedException(typeTag + " is not a supported spatial data type.");
}
}
+
+ public static IBinaryTokenizerFactory getBinaryTokenizerFactory(ATypeTag keyType, IndexType indexType,
+ int gramLength) throws AlgebricksException {
+ switch (indexType) {
+ case WORD_INVIX:
+ case FUZZY_WORD_INVIX: {
+ return AqlBinaryTokenizerFactoryProvider.INSTANCE.getWordTokenizerFactory(keyType, false);
+ }
+ case NGRAM_INVIX:
+ case FUZZY_NGRAM_INVIX: {
+ return AqlBinaryTokenizerFactoryProvider.INSTANCE.getNGramTokenizerFactory(keyType, gramLength, true,
+ false);
+ }
+ default: {
+ throw new AlgebricksException("Tokenizer not applicable to index type '" + indexType + "'.");
+ }
+ }
+ }
+
+ public static IBinaryComparatorFactory getTokenBinaryComparatorFactory(IAType keyType) throws AlgebricksException {
+ IAType type = keyType;
+ ATypeTag typeTag = keyType.getTypeTag();
+ // Extract item type from list.
+ if (typeTag == ATypeTag.UNORDEREDLIST || typeTag == ATypeTag.ORDEREDLIST) {
+ AbstractCollectionType listType = (AbstractCollectionType) keyType;
+ if (!listType.isTyped()) {
+ throw new AlgebricksException("Cannot build an inverted index on untyped lists.)");
+ }
+ type = listType.getItemType();
+ }
+ // Ignore case for string types.
+ return AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(type, true, true);
+ }
+
+ public static ITypeTraits getTokenTypeTrait(IAType keyType) throws AlgebricksException {
+ IAType type = keyType;
+ ATypeTag typeTag = keyType.getTypeTag();
+ // Extract item type from list.
+ if (typeTag == ATypeTag.UNORDEREDLIST) {
+ AUnorderedListType ulistType = (AUnorderedListType) keyType;
+ if (!ulistType.isTyped()) {
+ throw new AlgebricksException("Cannot build an inverted index on untyped lists.)");
+ }
+ type = ulistType.getItemType();
+ }
+ if (typeTag == ATypeTag.ORDEREDLIST) {
+ AOrderedListType olistType = (AOrderedListType) keyType;
+ if (!olistType.isTyped()) {
+ throw new AlgebricksException("Cannot build an inverted index on untyped lists.)");
+ }
+ type = olistType.getItemType();
+ }
+ return AqlTypeTraitProvider.INSTANCE.getTypeTrait(type);
+ }
}