[MULTIPLE ISSUES][COMP] Multiple array index fixes
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Fixes ASTERIXDB-3284 and ASTERIXDB-3285.
In the upsert pipeline, warning is returned instead of an error
when the indexed field is not an array. Elements of the
array having type different then the indexed type are not inserted
in the index. For the bulk load pipeline, on type mismatch the
index creation fails.
Change-Id: Icb7dd9ec54b4cf77963172adc93ed93704a2455d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17987
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Peeyush Gupta <peeyush.gupta@couchbase.com>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index 09ad4d1..446c779 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -690,7 +690,7 @@
int sourceIndicatorForBaseRecord = arrayIndexDetails.getElementList().get(0).getSourceIndicator();
LogicalVariable sourceVarForBaseRecord = hasMetaPart
? ((sourceIndicatorForBaseRecord == Index.RECORD_INDICATOR) ? recordVar : metaVar) : recordVar;
- UnnestBranchCreator branchCreator = new UnnestBranchCreator(sourceVarForBaseRecord, unnestSourceOp);
+ UnnestBranchCreator branchCreator = new UnnestBranchCreator(index, sourceVarForBaseRecord, unnestSourceOp);
Set<LogicalVariable> secondaryKeyVars = new LinkedHashSet<>();
for (Index.ArrayIndexElement workingElement : arrayIndexDetails.getElementList()) {
@@ -712,7 +712,12 @@
? getFieldAccessFunction(new MutableObject<>(varRef),
recordType.getFieldIndex(atomicFieldName.get(0)), atomicFieldName)
: getFieldAccessFunction(new MutableObject<>(varRef), -1, atomicFieldName);
-
+ IAType fieldType = recordType.getSubFieldType(atomicFieldName);
+ if (fieldType == null) {
+ newVarRef = castFunction(
+ index.isEnforced() ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX,
+ workingElement.getTypeList().get(0), newVarRef, sourceLoc);
+ }
// Add an assign on top to extract the atomic element.
AssignOperator newAssignOp = new AssignOperator(newVar, new MutableObject<>(newVarRef));
newAssignOp.setSourceLocation(sourceLoc);
@@ -728,16 +733,17 @@
workingElement.getUnnestList(), workingElement.getProjectList().get(0));
List<Boolean> firstUnnestFlags = ArrayIndexUtil.getUnnestFlags(workingElement.getUnnestList(),
workingElement.getProjectList().get(0));
- ArrayIndexUtil.walkArrayPath(index, recordType, flatFirstFieldName, firstUnnestFlags, branchCreator);
+ ArrayIndexUtil.walkArrayPath(index, workingElement, recordType, flatFirstFieldName, firstUnnestFlags,
+ branchCreator);
secondaryKeyVars.add(branchCreator.lastFieldVars.get(0));
// For all other elements in the PROJECT list, add an assign.
for (int j = 1; j < workingElement.getProjectList().size(); j++) {
LogicalVariable newVar = context.newVar();
- AbstractFunctionCallExpression newVarRef =
+ ILogicalExpression newVarRef =
getFieldAccessFunction(new MutableObject<>(branchCreator.createLastRecordVarRef()), -1,
workingElement.getProjectList().get(j));
-
+ newVarRef = createCastExpressionForArrayIndex(newVarRef, recordType, index, workingElement, j);
AssignOperator newAssignOp = new AssignOperator(newVar, new MutableObject<>(newVarRef));
newAssignOp.setSourceLocation(sourceLoc);
branchCreator.currentTop = introduceNewOp(branchCreator.currentTop, newAssignOp, true);
@@ -933,7 +939,7 @@
}
private ScalarFunctionCallExpression castFunction(FunctionIdentifier castFun, IAType requiredType,
- AbstractFunctionCallExpression inputExpr, SourceLocation sourceLoc) throws CompilationException {
+ ILogicalExpression inputExpr, SourceLocation sourceLoc) throws CompilationException {
BuiltinFunctionInfo castInfo = BuiltinFunctions.getBuiltinFunctionInfo(castFun);
ScalarFunctionCallExpression castExpr = new ScalarFunctionCallExpression(castInfo);
castExpr.setSourceLocation(sourceLoc);
@@ -957,6 +963,18 @@
return constructorExpr;
}
+ private ILogicalExpression createCastExpressionForArrayIndex(ILogicalExpression varRef, ARecordType recordType,
+ Index index, Index.ArrayIndexElement workingElement, int fieldPos) throws AlgebricksException {
+ IAType fieldType = ArrayIndexUtil.getSubFieldType(recordType, workingElement.getUnnestList(),
+ workingElement.getProjectList().get(fieldPos));
+ if (fieldType != null) {
+ return varRef;
+ } else {
+ return castFunction(index.isEnforced() ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX,
+ workingElement.getTypeList().get(fieldPos), varRef, sourceLoc);
+ }
+ }
+
private ILogicalOperator introduceNewOp(ILogicalOperator currentTopOp, ILogicalOperator newOp, boolean afterOp)
throws AlgebricksException {
if (afterOp) {
@@ -1092,8 +1110,10 @@
private final List<LogicalVariable> lastFieldVars;
private LogicalVariable lastRecordVar;
private ILogicalOperator currentTop, currentBottom = null;
+ private final Index index;
- public UnnestBranchCreator(LogicalVariable recordVar, ILogicalOperator sourceOperator) {
+ public UnnestBranchCreator(Index index, LogicalVariable recordVar, ILogicalOperator sourceOperator) {
+ this.index = index;
this.lastRecordVar = recordVar;
this.currentTop = sourceOperator;
this.lastFieldVars = new ArrayList<>();
@@ -1191,19 +1211,23 @@
}
@Override
- public void executeActionOnFinalArrayStep(ARecordType startingStepRecordType, List<String> fieldName,
- boolean isNonArrayStep, boolean requiresOnlyOneUnnest) throws AlgebricksException {
+ public void executeActionOnFinalArrayStep(Index.ArrayIndexElement workingElement, ARecordType baseRecordType,
+ ARecordType startingStepRecordType, List<String> fieldName, boolean isNonArrayStep,
+ boolean requiresOnlyOneUnnest) throws AlgebricksException {
// If the final value is nested inside a record, add an additional ASSIGN.
+ ILogicalExpression accessToFinalVar;
if (!isNonArrayStep) {
- return;
+ accessToFinalVar = createCastExpressionForArrayIndex(createLastRecordVarRef(), baseRecordType, index,
+ workingElement, 0);
+ } else {
+ // Create the function to access our final field.
+ accessToFinalVar = (startingStepRecordType != null)
+ ? getFieldAccessFunction(new MutableObject<>(createLastRecordVarRef()),
+ startingStepRecordType.getFieldIndex(fieldName.get(0)), fieldName)
+ : getFieldAccessFunction(new MutableObject<>(createLastRecordVarRef()), -1, fieldName);
+ accessToFinalVar =
+ createCastExpressionForArrayIndex(accessToFinalVar, baseRecordType, index, workingElement, 0);
}
-
- // Create the function to access our final field.
- AbstractFunctionCallExpression accessToFinalVar = (startingStepRecordType != null)
- ? getFieldAccessFunction(new MutableObject<>(createLastRecordVarRef()),
- startingStepRecordType.getFieldIndex(fieldName.get(0)), fieldName)
- : getFieldAccessFunction(new MutableObject<>(createLastRecordVarRef()), -1, fieldName);
-
LogicalVariable finalVar = context.newVar();
this.lastFieldVars.add(finalVar);
AssignOperator assignOperator = new AssignOperator(finalVar, new MutableObject<>(accessToFinalVar));
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index 25d42df..b66a0fd 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -3162,7 +3162,7 @@
List<String> flatName = ArrayIndexUtil.getFlattenedKeyFieldNames(e.getUnnestList(), project);
List<Boolean> unnestFlags = ArrayIndexUtil.getUnnestFlags(e.getUnnestList(), project);
analysisCtx.getArrayIndexStructureMatcher().reset(assignVar, subTree);
- ArrayIndexUtil.walkArrayPath(index, subTree.getRecordType(), flatName, unnestFlags,
+ ArrayIndexUtil.walkArrayPath(index, e, subTree.getRecordType(), flatName, unnestFlags,
analysisCtx.getArrayIndexStructureMatcher());
LogicalVariable varAfterWalk = analysisCtx.getArrayIndexStructureMatcher().getEndVar();
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/ArrayIndexStructureMatcher.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/ArrayIndexStructureMatcher.java
index 62b266a..ddfdcac 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/ArrayIndexStructureMatcher.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/ArrayIndexStructureMatcher.java
@@ -23,6 +23,7 @@
import java.util.Collections;
import java.util.List;
+import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.utils.ArrayIndexUtil;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.base.AString;
@@ -72,8 +73,9 @@
}
@Override
- public void executeActionOnFinalArrayStep(ARecordType startingStepRecordType, List<String> fieldName,
- boolean isNonArrayStep, boolean requiresOnlyOneUnnest) {
+ public void executeActionOnFinalArrayStep(Index.ArrayIndexElement workingElement, ARecordType baseRecordType,
+ ARecordType startingStepRecordType, List<String> fieldName, boolean isNonArrayStep,
+ boolean requiresOnlyOneUnnest) {
if (isNonArrayStep) {
isStructureMatched = isStructureMatched && matchAssignVarAndFieldName(startingStepRecordType, fieldName);
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.000.ddl.sqlpp
new file mode 100644
index 0000000..efe6d6b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.000.ddl.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test array indexes where some tuples have malformed values in indexed fields
+ * Expected Res : Said tuples are not indexed
+ */
+
+CREATE DATAVERSE test;
+USE test;
+CREATE TYPE dt1 as {id:int};
+CREATE DATASET ds1(dt1) primary key id;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.001.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.001.update.sqlpp
new file mode 100644
index 0000000..1822abe
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.001.update.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+UPSERT INTO ds1 {"id": 1, "a": ["hello", 1]};
+UPSERT INTO ds1 {"id": 2, "b": ["hello", "hello"]};
+UPSERT INTO ds1 {"id": 3, "b": [11, 12]};
+UPSERT INTO ds1 {"id": 4, "c": [10, 10]};
+
+UPSERT INTO ds1 {"id": 5, "d": [{"x":1}, {"x":"A"}]};
+UPSERT INTO ds1 {"id": 6, "d": [{"x":1}, {"x":"A"}]};
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.002.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.002.ddl.sqlpp
new file mode 100644
index 0000000..fbe85d3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.002.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+CREATE INDEX i2 ON ds1(UNNEST b:int) EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.003.ddl.sqlpp
new file mode 100644
index 0000000..fbe85d3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.003.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+CREATE INDEX i2 ON ds1(UNNEST b:int) EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.004.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.004.ddl.sqlpp
new file mode 100644
index 0000000..70119b5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.004.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+CREATE INDEX i2 ON ds1(UNNEST d select x:string) EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.005.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.005.ddl.sqlpp
new file mode 100644
index 0000000..ee3de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.005.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+CREATE INDEX i4 ON ds1(UNNEST c:int) EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp
new file mode 100644
index 0000000..dcf23cf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i4") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.999.ddl.sqlpp
new file mode 100644
index 0000000..86a1b59
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.000.ddl.sqlpp
new file mode 100644
index 0000000..e8af202
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.000.ddl.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test array indexes where some tuples have malformed values in indexed fields
+ * Expected Res : Said tuples are not indexed
+ */
+
+CREATE DATAVERSE test;
+USE test;
+CREATE TYPE dt1 as {id:int};
+CREATE DATASET ds1(dt1) primary key id;
+
+CREATE INDEX i1 ON ds1(UNNEST a : string) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i2 ON ds1(UNNEST b SELECT x : int) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i3 ON ds1(UNNEST b.x SELECT p: int) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i4 ON ds1(UNNEST b.c UNNEST d.e UNNEST t SELECT x.y : string, q.w: string, u: int) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i5 ON ds1(p: int, (UNNEST b.c UNNEST d.e UNNEST t SELECT x.y : string, q.w: string), z.m: double) EXCLUDE UNKNOWN KEY;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.001.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.001.update.sqlpp
new file mode 100644
index 0000000..9c2b805
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.001.update.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+UPSERT INTO ds1 {"id": 1, "a": [10, "hello", 10, "hello"]};
+UPSERT INTO ds1 {"id": 2, "a": [10]};
+UPSERT INTO ds1 {"id": 3, "a": 94};
+UPSERT INTO ds1 {"id": 4, "a": {"x":1}};
+
+UPSERT INTO ds1 {"id": 5, "a": ["hello"], "b": [{"x":1}, {"x":"bb"}]};
+UPSERT INTO ds1 {"id": 6, "a": ["hello"], "b": [{"x":"aa"}]};
+UPSERT INTO ds1 {"id": 7, "a": 10, "b": [{"x":10}]};
+UPSERT INTO ds1 {"id": 8, "a": ["hello"], "b":100};
+UPSERT INTO ds1 {"id": 9, "b": 100};
+
+UPSERT INTO ds1 {"id": 10, "a": ["hello"], "b":{"x":[{"p":1}, {"k":1}]}};
+
+UPSERT INTO ds1 {"id": 11, "a": ["hello"], "b": {"c":[{"d": {"e":[{"t":[{"x":{"y":"aab"}, "q":{"w":"10"}, "u":90}]}]}}]}};
+UPSERT INTO ds1 {"id": 12, "a": ["hello"], "b": {"c":[{"d": {"e":[{"t":{"x":{"y":"aab"}, "q":{"w":"10"}, "u":91}}]}}]}};
+UPSERT INTO ds1 {"id": 13, "a": ["hello"], "b": {"c":[{"d": {"e":{"t":[{"x":{"y":"aab"}, "q":{"w":"10"}, "u":92}]}}}]}};
+
+UPSERT INTO ds1 {"id": 14, "a": ["hello"], "b":{"c":[{"d":{"e":[{"t":[{"x":{"y":"aab"}, "q":{"w":"10"}, "u":93}]}]}}]}, "p":100, "z":{"m":100.10}};
+UPSERT INTO ds1 {"id": 15, "a": ["hello"], "b":{"c":[{"d":{"e":[{"t":[{"x":{"y":"aab"}, "q":{"w":"10"}}]}]}}]}, "p":"kk", "z":{"m":100.10}};
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.002.query.sqlpp
new file mode 100644
index 0000000..e1bb086
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.002.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT count(*) from ds1;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.003.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.003.query.sqlpp
new file mode 100644
index 0000000..18132bb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.003.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i1") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.004.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.004.query.sqlpp
new file mode 100644
index 0000000..afb1d61
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.004.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i2") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.005.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.005.query.sqlpp
new file mode 100644
index 0000000..57ad1d9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.005.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i3") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.006.query.sqlpp
new file mode 100644
index 0000000..dcf23cf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.006.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i4") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.007.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.007.query.sqlpp
new file mode 100644
index 0000000..b070cbc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.007.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i5") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.999.ddl.sqlpp
new file mode 100644
index 0000000..86a1b59
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields-bulkload/index-bad-fields.006.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields-bulkload/index-bad-fields.006.adm
new file mode 100644
index 0000000..917cc17
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields-bulkload/index-bad-fields.006.adm
@@ -0,0 +1 @@
+{ "values": [ 10, 4 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.002.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.002.adm
new file mode 100644
index 0000000..746d306
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.002.adm
@@ -0,0 +1 @@
+{ "$1": 15 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.003.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.003.adm
new file mode 100644
index 0000000..a464380
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.003.adm
@@ -0,0 +1,10 @@
+{ "values": [ "hello", 1 ] }
+{ "values": [ "hello", 5 ] }
+{ "values": [ "hello", 6 ] }
+{ "values": [ "hello", 8 ] }
+{ "values": [ "hello", 10 ] }
+{ "values": [ "hello", 11 ] }
+{ "values": [ "hello", 12 ] }
+{ "values": [ "hello", 13 ] }
+{ "values": [ "hello", 14 ] }
+{ "values": [ "hello", 15 ] }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.004.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.004.adm
new file mode 100644
index 0000000..3fdbd81
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.004.adm
@@ -0,0 +1,2 @@
+{ "values": [ 1, 5 ] }
+{ "values": [ 10, 7 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.005.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.005.adm
new file mode 100644
index 0000000..6045d06
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.005.adm
@@ -0,0 +1 @@
+{ "values": [ 1, 10 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.006.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.006.adm
new file mode 100644
index 0000000..4c44dba
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.006.adm
@@ -0,0 +1,2 @@
+{ "values": [ "aab", "10", 90, 11 ] }
+{ "values": [ "aab", "10", 93, 14 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.007.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.007.adm
new file mode 100644
index 0000000..a38bbc6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.007.adm
@@ -0,0 +1 @@
+{ "values": [ 100, "aab", "10", 100.1, 14 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/q06/q06.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/q06/q06.3.adm
new file mode 100644
index 0000000..21c2ffa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/q06/q06.3.adm
@@ -0,0 +1,10 @@
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/sugar-01-negative/core-01.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/sugar-01-negative/core-01.1.adm
new file mode 100644
index 0000000..cc15b26
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/sugar-01-negative/core-01.1.adm
@@ -0,0 +1 @@
+{ "avgpay": null, "workers": [ { "name": "Bill", "salary": 2000 }, { "name": "Fred", "salary": 3000 } ], "deptno": "K55" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2550/query-ASTERIXDB-2886.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2550/query-ASTERIXDB-2886.3.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2550/query-ASTERIXDB-2886.3.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 0ab9672..dc31f7e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -5801,8 +5801,7 @@
</test-case>
<test-case FilePath="global-aggregate">
<compilation-unit name="q06_error">
- <output-dir compare="Text">q01</output-dir>
- <expected-error>ASX0037: Type mismatch: expected value of type array or multiset, but got the value of type string (in line 22, at column 8)</expected-error>
+ <output-dir compare="Text">q06</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="global-aggregate">
@@ -5939,8 +5938,7 @@
</test-case>
<test-case FilePath="group-by">
<compilation-unit name="sugar-01-negative">
- <output-dir compare="Text">core-01</output-dir>
- <expected-error>ASX0037: Type mismatch: expected value of type array or multiset, but got the value of type bigint (in line 26, at column 26)</expected-error>
+ <output-dir compare="Text">sugar-01-negative</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="group-by">
@@ -7164,8 +7162,7 @@
</test-case>
<test-case FilePath="misc">
<compilation-unit name="query-ASTERIXDB-2550">
- <output-dir compare="Text">none</output-dir>
- <expected-error>ASX0037: Type mismatch: expected value of type array or multiset, but got the value of type object (in line 28, at column 2)</expected-error>
+ <output-dir compare="Text">query-ASTERIXDB-2550</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="misc">
@@ -8359,6 +8356,8 @@
<test-case FilePath="array-index/bulk-loading/on-index-creation/open">
<compilation-unit name="composite-atomic">
<output-dir compare="Text">composite-atomic</output-dir>
+ <expected-error>ASX0001: Field type string cannot be promoted to type bigint</expected-error>
+ <source-location>false</source-location>
</compilation-unit>
</test-case>
</test-group>
@@ -8656,6 +8655,24 @@
</compilation-unit>
</test-case>
</test-group>
+ <test-group name="array-index/index-bad-fields">
+ <test-case FilePath="array-index">
+ <compilation-unit name="index-bad-fields">
+ <output-dir compare="Text">index-bad-fields</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/index-bad-fields-bulkload">
+ <test-case FilePath="array-index">
+ <compilation-unit name="index-bad-fields-bulkload">
+ <output-dir compare="Text">index-bad-fields-bulkload</output-dir>
+ <expected-error>ASX0018: Cannot convert the string type to the bigint type</expected-error>
+ <expected-error>ASX0018: Cannot convert the string type to the bigint type</expected-error>
+ <expected-error>ASX0001: Field type bigint cannot be promoted to type string</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ </test-group>
</test-group>
<test-group name="nestrecords">
<test-case FilePath="nestrecords">
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
index 1abf300..7332dd0 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
@@ -240,8 +240,9 @@
* Traverse each distinct record path and invoke the appropriate commands for each scenario. Here, we keep track
* of the record/list type at each step and give this to each command.
*/
- public static void walkArrayPath(Index index, ARecordType baseRecordType, List<String> flattenedFieldName,
- List<Boolean> unnestFlags, TypeTrackerCommandExecutor commandExecutor) throws AlgebricksException {
+ public static void walkArrayPath(Index index, Index.ArrayIndexElement workingElement, ARecordType baseRecordType,
+ List<String> flattenedFieldName, List<Boolean> unnestFlags, TypeTrackerCommandExecutor commandExecutor)
+ throws AlgebricksException {
ArrayPath arrayPath = new ArrayPath(flattenedFieldName, unnestFlags).invoke();
List<List<String>> fieldNamesPerArray = arrayPath.fieldNamesPerArray;
List<Boolean> unnestFlagsPerArray = arrayPath.unnestFlagsPerArray;
@@ -286,15 +287,15 @@
}
}
boolean isFirstArrayStep = i == 0;
- boolean isLastUnnestInIntermediateStep = i < fieldNamesPerArray.size() - 1;
+ boolean isLastUnnestInIntermediateStep = i <= fieldNamesPerArray.size() - 1;
commandExecutor.executeActionOnEachArrayStep(startingStepRecordType, workingType,
fieldNamesPerArray.get(i), isFirstArrayStep, isLastUnnestInIntermediateStep);
}
if (i == fieldNamesPerArray.size() - 1) {
boolean isNonArrayStep = !unnestFlagsPerArray.get(i);
- commandExecutor.executeActionOnFinalArrayStep(startingStepRecordType, fieldNamesPerArray.get(i),
- isNonArrayStep, requiresOnlyOneUnnest);
+ commandExecutor.executeActionOnFinalArrayStep(workingElement, baseRecordType, startingStepRecordType,
+ fieldNamesPerArray.get(i), isNonArrayStep, requiresOnlyOneUnnest);
}
}
}
@@ -341,8 +342,9 @@
List<String> fieldName, boolean isFirstArrayStep, boolean isLastUnnestInIntermediateStep)
throws AlgebricksException;
- void executeActionOnFinalArrayStep(ARecordType startingStepRecordType, List<String> fieldName,
- boolean isNonArrayStep, boolean requiresOnlyOneUnnest) throws AlgebricksException;
+ void executeActionOnFinalArrayStep(Index.ArrayIndexElement workingElement, ARecordType baseRecordType,
+ ARecordType startingStepRecordType, List<String> fieldName, boolean isNonArrayStep,
+ boolean requiresOnlyOneUnnest) throws AlgebricksException;
}
private static class ArrayPath {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
index c8a7ee1..af0fd82 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
@@ -158,7 +158,7 @@
int flattenedListPos = 0;
for (Index.ArrayIndexElement e : arrayIndexDetails.getElementList()) {
for (int i = 0; i < e.getProjectList().size(); i++) {
- addSKEvalFactories(isOverridingKeyFieldTypes ? enforcedItemType : itemType, flattenedListPos, false);
+ addSKEvalFactories(isOverridingKeyFieldTypes ? enforcedItemType : itemType, flattenedListPos, false, e);
Pair<IAType, Boolean> keyTypePair = ArrayIndexUtil.getNonNullableOpenFieldType(e.getTypeList().get(i),
e.getUnnestList(), e.getProjectList().get(i), itemType);
IAType keyType = keyTypePair.first;
@@ -203,7 +203,7 @@
ARecordType filterItemType =
((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterSourceIndicator() == 0 ? itemType
: metaType;
- addSKEvalFactories(itemType, numSecondaryKeys, true);
+ addSKEvalFactories(itemType, numSecondaryKeys, true, null);
Pair<IAType, Boolean> keyTypePair;
keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, filterItemType);
IAType type = keyTypePair.first;
@@ -230,8 +230,8 @@
return fieldPermutation;
}
- protected void addSKEvalFactories(ARecordType recordType, int fieldPos, boolean isFilterField)
- throws AlgebricksException {
+ protected void addSKEvalFactories(ARecordType recordType, int fieldPos, boolean isFilterField,
+ Index.ArrayIndexElement workingElement) throws AlgebricksException {
if (isFilterField) {
addFilterFieldToBuilder(recordType);
return;
@@ -245,7 +245,8 @@
} else {
EvalFactoryAndRecDescInvoker commandExecutor =
new EvalFactoryAndRecDescInvoker(!evalFactoryAndRecDescStackBuilder.isUnnestEvalPopulated());
- ArrayIndexUtil.walkArrayPath(index, recordType, flattenedFieldName, workingUnnestFlags, commandExecutor);
+ ArrayIndexUtil.walkArrayPath(index, workingElement, recordType, flattenedFieldName, workingUnnestFlags,
+ commandExecutor);
}
}
@@ -269,7 +270,7 @@
sourceOp = targetOp;
if (arrayIndexDetails.isOverridingKeyFieldTypes() && !enforcedItemType.equals(itemType)) {
// If we have an enforced type, insert a "cast" after the primary index scan.
- targetOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced());
+ targetOp = createCastOp(spec, dataset.getDatasetType(), true);
spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
sourceOp = targetOp;
}
@@ -556,8 +557,9 @@
}
@Override
- public void executeActionOnFinalArrayStep(ARecordType startingStepRecordType, List<String> fieldName,
- boolean isNonArrayStep, boolean requiresOnlyOneUnnest) throws AlgebricksException {
+ public void executeActionOnFinalArrayStep(Index.ArrayIndexElement workingElement, ARecordType baseRecordType,
+ ARecordType startingStepRecordType, List<String> fieldName, boolean isNonArrayStep,
+ boolean requiresOnlyOneUnnest) throws AlgebricksException {
// If the final value is nested inside a record, add this SEF.
if (!isNonArrayStep) {
return;
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/scalar/AbstractScalarAggregateDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/scalar/AbstractScalarAggregateDescriptor.java
index 064c861..2fd065d 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/scalar/AbstractScalarAggregateDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/scalar/AbstractScalarAggregateDescriptor.java
@@ -80,7 +80,7 @@
public IScalarEvaluator createScalarEvaluator(IEvaluatorContext ctx) throws HyracksDataException {
// Use ScanCollection to iterate over list items.
ScanCollectionUnnestingFunctionFactory scanCollectionFactory =
- new ScanCollectionUnnestingFunctionFactory(args[0], sourceLoc);
+ new ScanCollectionUnnestingFunctionFactory(args[0], sourceLoc, getIdentifier());
return createScalarAggregateEvaluator(aggFuncFactory.createAggregateEvaluator(ctx),
scanCollectionFactory, ctx);
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/unnestingfunctions/std/ScanCollectionDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/unnestingfunctions/std/ScanCollectionDescriptor.java
index ff6dd89..6f9dd5c 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/unnestingfunctions/std/ScanCollectionDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/unnestingfunctions/std/ScanCollectionDescriptor.java
@@ -21,11 +21,11 @@
import java.io.IOException;
+import org.apache.asterix.om.exceptions.ExceptionUtil;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.runtime.evaluators.common.ListAccessor;
-import org.apache.asterix.runtime.exceptions.TypeMismatchException;
import org.apache.asterix.runtime.unnestingfunctions.base.AbstractUnnestingFunctionDynamicDescriptor;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
@@ -52,7 +52,7 @@
@Override
public IUnnestingEvaluatorFactory createUnnestingEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new ScanCollectionUnnestingFunctionFactory(args[0], sourceLoc);
+ return new ScanCollectionUnnestingFunctionFactory(args[0], sourceLoc, getIdentifier());
}
public static class ScanCollectionUnnestingFunctionFactory implements IUnnestingEvaluatorFactory {
@@ -60,10 +60,13 @@
private static final long serialVersionUID = 1L;
private IScalarEvaluatorFactory listEvalFactory;
private final SourceLocation sourceLoc;
+ private final FunctionIdentifier funID;
- public ScanCollectionUnnestingFunctionFactory(IScalarEvaluatorFactory arg, SourceLocation sourceLoc) {
+ public ScanCollectionUnnestingFunctionFactory(IScalarEvaluatorFactory arg, SourceLocation sourceLoc,
+ FunctionIdentifier funID) {
this.listEvalFactory = arg;
this.sourceLoc = sourceLoc;
+ this.funID = funID;
}
@Override
@@ -88,8 +91,9 @@
}
if (typeTag != ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
&& typeTag != ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
- throw new TypeMismatchException(sourceLoc, typeTag, ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG,
- ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
+ ExceptionUtil.warnTypeMismatch(ctx, sourceLoc, funID, typeTag, 0, ATypeTag.MULTISET);
+ metUnknown = true;
+ return;
}
listAccessor.reset(inputVal.getByteArray(), inputVal.getStartOffset());
itemIndex = 0;