Merge branch 'gerrit/neo' into 'gerrit/trinity'
Change-Id: Ia94fc0878d6468495233cb06268132fdee71b7f1
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index 09ad4d1..446c779 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -690,7 +690,7 @@
int sourceIndicatorForBaseRecord = arrayIndexDetails.getElementList().get(0).getSourceIndicator();
LogicalVariable sourceVarForBaseRecord = hasMetaPart
? ((sourceIndicatorForBaseRecord == Index.RECORD_INDICATOR) ? recordVar : metaVar) : recordVar;
- UnnestBranchCreator branchCreator = new UnnestBranchCreator(sourceVarForBaseRecord, unnestSourceOp);
+ UnnestBranchCreator branchCreator = new UnnestBranchCreator(index, sourceVarForBaseRecord, unnestSourceOp);
Set<LogicalVariable> secondaryKeyVars = new LinkedHashSet<>();
for (Index.ArrayIndexElement workingElement : arrayIndexDetails.getElementList()) {
@@ -712,7 +712,12 @@
? getFieldAccessFunction(new MutableObject<>(varRef),
recordType.getFieldIndex(atomicFieldName.get(0)), atomicFieldName)
: getFieldAccessFunction(new MutableObject<>(varRef), -1, atomicFieldName);
-
+ IAType fieldType = recordType.getSubFieldType(atomicFieldName);
+ if (fieldType == null) {
+ newVarRef = castFunction(
+ index.isEnforced() ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX,
+ workingElement.getTypeList().get(0), newVarRef, sourceLoc);
+ }
// Add an assign on top to extract the atomic element.
AssignOperator newAssignOp = new AssignOperator(newVar, new MutableObject<>(newVarRef));
newAssignOp.setSourceLocation(sourceLoc);
@@ -728,16 +733,17 @@
workingElement.getUnnestList(), workingElement.getProjectList().get(0));
List<Boolean> firstUnnestFlags = ArrayIndexUtil.getUnnestFlags(workingElement.getUnnestList(),
workingElement.getProjectList().get(0));
- ArrayIndexUtil.walkArrayPath(index, recordType, flatFirstFieldName, firstUnnestFlags, branchCreator);
+ ArrayIndexUtil.walkArrayPath(index, workingElement, recordType, flatFirstFieldName, firstUnnestFlags,
+ branchCreator);
secondaryKeyVars.add(branchCreator.lastFieldVars.get(0));
// For all other elements in the PROJECT list, add an assign.
for (int j = 1; j < workingElement.getProjectList().size(); j++) {
LogicalVariable newVar = context.newVar();
- AbstractFunctionCallExpression newVarRef =
+ ILogicalExpression newVarRef =
getFieldAccessFunction(new MutableObject<>(branchCreator.createLastRecordVarRef()), -1,
workingElement.getProjectList().get(j));
-
+ newVarRef = createCastExpressionForArrayIndex(newVarRef, recordType, index, workingElement, j);
AssignOperator newAssignOp = new AssignOperator(newVar, new MutableObject<>(newVarRef));
newAssignOp.setSourceLocation(sourceLoc);
branchCreator.currentTop = introduceNewOp(branchCreator.currentTop, newAssignOp, true);
@@ -933,7 +939,7 @@
}
private ScalarFunctionCallExpression castFunction(FunctionIdentifier castFun, IAType requiredType,
- AbstractFunctionCallExpression inputExpr, SourceLocation sourceLoc) throws CompilationException {
+ ILogicalExpression inputExpr, SourceLocation sourceLoc) throws CompilationException {
BuiltinFunctionInfo castInfo = BuiltinFunctions.getBuiltinFunctionInfo(castFun);
ScalarFunctionCallExpression castExpr = new ScalarFunctionCallExpression(castInfo);
castExpr.setSourceLocation(sourceLoc);
@@ -957,6 +963,18 @@
return constructorExpr;
}
+ private ILogicalExpression createCastExpressionForArrayIndex(ILogicalExpression varRef, ARecordType recordType,
+ Index index, Index.ArrayIndexElement workingElement, int fieldPos) throws AlgebricksException {
+ IAType fieldType = ArrayIndexUtil.getSubFieldType(recordType, workingElement.getUnnestList(),
+ workingElement.getProjectList().get(fieldPos));
+ if (fieldType != null) {
+ return varRef;
+ } else {
+ return castFunction(index.isEnforced() ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX,
+ workingElement.getTypeList().get(fieldPos), varRef, sourceLoc);
+ }
+ }
+
private ILogicalOperator introduceNewOp(ILogicalOperator currentTopOp, ILogicalOperator newOp, boolean afterOp)
throws AlgebricksException {
if (afterOp) {
@@ -1092,8 +1110,10 @@
private final List<LogicalVariable> lastFieldVars;
private LogicalVariable lastRecordVar;
private ILogicalOperator currentTop, currentBottom = null;
+ private final Index index;
- public UnnestBranchCreator(LogicalVariable recordVar, ILogicalOperator sourceOperator) {
+ public UnnestBranchCreator(Index index, LogicalVariable recordVar, ILogicalOperator sourceOperator) {
+ this.index = index;
this.lastRecordVar = recordVar;
this.currentTop = sourceOperator;
this.lastFieldVars = new ArrayList<>();
@@ -1191,19 +1211,23 @@
}
@Override
- public void executeActionOnFinalArrayStep(ARecordType startingStepRecordType, List<String> fieldName,
- boolean isNonArrayStep, boolean requiresOnlyOneUnnest) throws AlgebricksException {
+ public void executeActionOnFinalArrayStep(Index.ArrayIndexElement workingElement, ARecordType baseRecordType,
+ ARecordType startingStepRecordType, List<String> fieldName, boolean isNonArrayStep,
+ boolean requiresOnlyOneUnnest) throws AlgebricksException {
// If the final value is nested inside a record, add an additional ASSIGN.
+ ILogicalExpression accessToFinalVar;
if (!isNonArrayStep) {
- return;
+ accessToFinalVar = createCastExpressionForArrayIndex(createLastRecordVarRef(), baseRecordType, index,
+ workingElement, 0);
+ } else {
+ // Create the function to access our final field.
+ accessToFinalVar = (startingStepRecordType != null)
+ ? getFieldAccessFunction(new MutableObject<>(createLastRecordVarRef()),
+ startingStepRecordType.getFieldIndex(fieldName.get(0)), fieldName)
+ : getFieldAccessFunction(new MutableObject<>(createLastRecordVarRef()), -1, fieldName);
+ accessToFinalVar =
+ createCastExpressionForArrayIndex(accessToFinalVar, baseRecordType, index, workingElement, 0);
}
-
- // Create the function to access our final field.
- AbstractFunctionCallExpression accessToFinalVar = (startingStepRecordType != null)
- ? getFieldAccessFunction(new MutableObject<>(createLastRecordVarRef()),
- startingStepRecordType.getFieldIndex(fieldName.get(0)), fieldName)
- : getFieldAccessFunction(new MutableObject<>(createLastRecordVarRef()), -1, fieldName);
-
LogicalVariable finalVar = context.newVar();
this.lastFieldVars.add(finalVar);
AssignOperator assignOperator = new AssignOperator(finalVar, new MutableObject<>(accessToFinalVar));
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index 9c69902..10b763b 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -3222,7 +3222,7 @@
List<String> flatName = ArrayIndexUtil.getFlattenedKeyFieldNames(e.getUnnestList(), project);
List<Boolean> unnestFlags = ArrayIndexUtil.getUnnestFlags(e.getUnnestList(), project);
analysisCtx.getArrayIndexStructureMatcher().reset(assignVar, subTree);
- ArrayIndexUtil.walkArrayPath(index, subTree.getRecordType(), flatName, unnestFlags,
+ ArrayIndexUtil.walkArrayPath(index, e, subTree.getRecordType(), flatName, unnestFlags,
analysisCtx.getArrayIndexStructureMatcher());
LogicalVariable varAfterWalk = analysisCtx.getArrayIndexStructureMatcher().getEndVar();
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/ArrayIndexStructureMatcher.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/ArrayIndexStructureMatcher.java
index 62b266a..ddfdcac 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/ArrayIndexStructureMatcher.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/ArrayIndexStructureMatcher.java
@@ -23,6 +23,7 @@
import java.util.Collections;
import java.util.List;
+import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.utils.ArrayIndexUtil;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.base.AString;
@@ -72,8 +73,9 @@
}
@Override
- public void executeActionOnFinalArrayStep(ARecordType startingStepRecordType, List<String> fieldName,
- boolean isNonArrayStep, boolean requiresOnlyOneUnnest) {
+ public void executeActionOnFinalArrayStep(Index.ArrayIndexElement workingElement, ARecordType baseRecordType,
+ ARecordType startingStepRecordType, List<String> fieldName, boolean isNonArrayStep,
+ boolean requiresOnlyOneUnnest) {
if (isNonArrayStep) {
isStructureMatched = isStructureMatched && matchAssignVarAndFieldName(startingStepRecordType, fieldName);
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.000.ddl.sqlpp
new file mode 100644
index 0000000..e8af202
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.000.ddl.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test array indexes where some tuples have malformed values in indexed fields
+ * Expected Res : Said tuples are not indexed
+ */
+
+CREATE DATAVERSE test;
+USE test;
+CREATE TYPE dt1 as {id:int};
+CREATE DATASET ds1(dt1) primary key id;
+
+CREATE INDEX i1 ON ds1(UNNEST a : string) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i2 ON ds1(UNNEST b SELECT x : int) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i3 ON ds1(UNNEST b.x SELECT p: int) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i4 ON ds1(UNNEST b.c UNNEST d.e UNNEST t SELECT x.y : string, q.w: string, u: int) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i5 ON ds1(p: int, (UNNEST b.c UNNEST d.e UNNEST t SELECT x.y : string, q.w: string), z.m: double) EXCLUDE UNKNOWN KEY;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.001.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.001.update.sqlpp
new file mode 100644
index 0000000..9c2b805
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.001.update.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+UPSERT INTO ds1 {"id": 1, "a": [10, "hello", 10, "hello"]};
+UPSERT INTO ds1 {"id": 2, "a": [10]};
+UPSERT INTO ds1 {"id": 3, "a": 94};
+UPSERT INTO ds1 {"id": 4, "a": {"x":1}};
+
+UPSERT INTO ds1 {"id": 5, "a": ["hello"], "b": [{"x":1}, {"x":"bb"}]};
+UPSERT INTO ds1 {"id": 6, "a": ["hello"], "b": [{"x":"aa"}]};
+UPSERT INTO ds1 {"id": 7, "a": 10, "b": [{"x":10}]};
+UPSERT INTO ds1 {"id": 8, "a": ["hello"], "b":100};
+UPSERT INTO ds1 {"id": 9, "b": 100};
+
+UPSERT INTO ds1 {"id": 10, "a": ["hello"], "b":{"x":[{"p":1}, {"k":1}]}};
+
+UPSERT INTO ds1 {"id": 11, "a": ["hello"], "b": {"c":[{"d": {"e":[{"t":[{"x":{"y":"aab"}, "q":{"w":"10"}, "u":90}]}]}}]}};
+UPSERT INTO ds1 {"id": 12, "a": ["hello"], "b": {"c":[{"d": {"e":[{"t":{"x":{"y":"aab"}, "q":{"w":"10"}, "u":91}}]}}]}};
+UPSERT INTO ds1 {"id": 13, "a": ["hello"], "b": {"c":[{"d": {"e":{"t":[{"x":{"y":"aab"}, "q":{"w":"10"}, "u":92}]}}}]}};
+
+UPSERT INTO ds1 {"id": 14, "a": ["hello"], "b":{"c":[{"d":{"e":[{"t":[{"x":{"y":"aab"}, "q":{"w":"10"}, "u":93}]}]}}]}, "p":100, "z":{"m":100.10}};
+UPSERT INTO ds1 {"id": 15, "a": ["hello"], "b":{"c":[{"d":{"e":[{"t":[{"x":{"y":"aab"}, "q":{"w":"10"}}]}]}}]}, "p":"kk", "z":{"m":100.10}};
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.002.query.sqlpp
new file mode 100644
index 0000000..e1bb086
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.002.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT count(*) from ds1;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.003.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.003.query.sqlpp
new file mode 100644
index 0000000..18132bb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.003.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i1") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.004.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.004.query.sqlpp
new file mode 100644
index 0000000..afb1d61
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.004.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i2") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.005.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.005.query.sqlpp
new file mode 100644
index 0000000..57ad1d9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.005.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i3") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.006.query.sqlpp
new file mode 100644
index 0000000..dcf23cf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.006.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i4") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.007.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.007.query.sqlpp
new file mode 100644
index 0000000..b070cbc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.007.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i5") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.008.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.008.ddl.sqlpp
new file mode 100644
index 0000000..7850fde
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.008.ddl.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test array indexes where some tuples have malformed values in indexed fields
+ * Expected Res : Said tuples are not indexed
+ */
+
+USE test;
+
+CREATE INDEX i6 ON ds1(UNNEST a : string) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i7 ON ds1(UNNEST b SELECT x : int) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i8 ON ds1(UNNEST b.x SELECT p: int) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i9 ON ds1(UNNEST b.c UNNEST d.e UNNEST t SELECT x.y : string, q.w: string, u: int) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i10 ON ds1(p: int, (UNNEST b.c UNNEST d.e UNNEST t SELECT x.y : string, q.w: string), z.m: double) EXCLUDE UNKNOWN KEY;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.009.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.009.query.sqlpp
new file mode 100644
index 0000000..a33bb23
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.009.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i6") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.010.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.010.query.sqlpp
new file mode 100644
index 0000000..4134554
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.010.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i7") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.011.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.011.query.sqlpp
new file mode 100644
index 0000000..75b56d1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.011.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i8") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.012.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.012.query.sqlpp
new file mode 100644
index 0000000..5364ebb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.012.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i9") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.013.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.013.query.sqlpp
new file mode 100644
index 0000000..44f59c8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.013.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i10") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.999.ddl.sqlpp
new file mode 100644
index 0000000..86a1b59
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/string/invalid-unicode/test.000.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/string/invalid-unicode/test.000.query.sqlpp
new file mode 100644
index 0000000..a533d7e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/string/invalid-unicode/test.000.query.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// param max-warnings:json=1000
+
+[
+ string_length("\uDEAD x \uDEAD"),
+ string_to_codepoint("\uDEAD x \uDEAD"),
+ trim("\uDEAD x \uDEAD"),
+ ltrim("\uDEAD x \uDEAD"),
+ rtrim("\uDEAD x \uDEAD"),
+ trim("\uDEAD x \uDEAD", "x"),
+ ltrim("\uDEAD x \uDEAD", "x"),
+ rtrim("\uDEAD x \uDEAD", "x"),
+ reverse("\uDEAD x \uDEAD"),
+ position("\uDEAD x \uDEAD", "x"),
+ position1("\uDEAD x \uDEAD", "x")
+];
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields-bulkload/index-bad-fields.006.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields-bulkload/index-bad-fields.006.adm
new file mode 100644
index 0000000..917cc17
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields-bulkload/index-bad-fields.006.adm
@@ -0,0 +1 @@
+{ "values": [ 10, 4 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.002.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.002.adm
new file mode 100644
index 0000000..746d306
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.002.adm
@@ -0,0 +1 @@
+{ "$1": 15 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.003.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.003.adm
new file mode 100644
index 0000000..a464380
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.003.adm
@@ -0,0 +1,10 @@
+{ "values": [ "hello", 1 ] }
+{ "values": [ "hello", 5 ] }
+{ "values": [ "hello", 6 ] }
+{ "values": [ "hello", 8 ] }
+{ "values": [ "hello", 10 ] }
+{ "values": [ "hello", 11 ] }
+{ "values": [ "hello", 12 ] }
+{ "values": [ "hello", 13 ] }
+{ "values": [ "hello", 14 ] }
+{ "values": [ "hello", 15 ] }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.004.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.004.adm
new file mode 100644
index 0000000..3fdbd81
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.004.adm
@@ -0,0 +1,2 @@
+{ "values": [ 1, 5 ] }
+{ "values": [ 10, 7 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.005.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.005.adm
new file mode 100644
index 0000000..6045d06
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.005.adm
@@ -0,0 +1 @@
+{ "values": [ 1, 10 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.006.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.006.adm
new file mode 100644
index 0000000..4c44dba
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.006.adm
@@ -0,0 +1,2 @@
+{ "values": [ "aab", "10", 90, 11 ] }
+{ "values": [ "aab", "10", 93, 14 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.007.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.007.adm
new file mode 100644
index 0000000..a38bbc6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.007.adm
@@ -0,0 +1 @@
+{ "values": [ 100, "aab", "10", 100.1, 14 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.009.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.009.adm
new file mode 100644
index 0000000..a464380
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.009.adm
@@ -0,0 +1,10 @@
+{ "values": [ "hello", 1 ] }
+{ "values": [ "hello", 5 ] }
+{ "values": [ "hello", 6 ] }
+{ "values": [ "hello", 8 ] }
+{ "values": [ "hello", 10 ] }
+{ "values": [ "hello", 11 ] }
+{ "values": [ "hello", 12 ] }
+{ "values": [ "hello", 13 ] }
+{ "values": [ "hello", 14 ] }
+{ "values": [ "hello", 15 ] }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.010.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.010.adm
new file mode 100644
index 0000000..3fdbd81
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.010.adm
@@ -0,0 +1,2 @@
+{ "values": [ 1, 5 ] }
+{ "values": [ 10, 7 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.011.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.011.adm
new file mode 100644
index 0000000..6045d06
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.011.adm
@@ -0,0 +1 @@
+{ "values": [ 1, 10 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.012.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.012.adm
new file mode 100644
index 0000000..4c44dba
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.012.adm
@@ -0,0 +1,2 @@
+{ "values": [ "aab", "10", 90, 11 ] }
+{ "values": [ "aab", "10", 93, 14 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.013.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.013.adm
new file mode 100644
index 0000000..a38bbc6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.013.adm
@@ -0,0 +1 @@
+{ "values": [ 100, "aab", "10", 100.1, 14 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/q06/q06.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/q06/q06.3.adm
new file mode 100644
index 0000000..21c2ffa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/q06/q06.3.adm
@@ -0,0 +1,10 @@
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/sugar-01-negative/core-01.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/sugar-01-negative/core-01.1.adm
new file mode 100644
index 0000000..cc15b26
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/sugar-01-negative/core-01.1.adm
@@ -0,0 +1 @@
+{ "avgpay": null, "workers": [ { "name": "Bill", "salary": 2000 }, { "name": "Fred", "salary": 3000 } ], "deptno": "K55" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2550/query-ASTERIXDB-2886.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2550/query-ASTERIXDB-2886.3.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2550/query-ASTERIXDB-2886.3.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/string/invalid-unicode/result.000.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/string/invalid-unicode/result.000.adm
new file mode 100644
index 0000000..a9fc274
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/string/invalid-unicode/result.000.adm
@@ -0,0 +1 @@
+[ null, null, null, null, null, null, null, null, null, null, null ]
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 4bc82f4..e3fe5710 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -5821,8 +5821,7 @@
</test-case>
<test-case FilePath="global-aggregate">
<compilation-unit name="q06_error">
- <output-dir compare="Text">q01</output-dir>
- <expected-error>ASX0037: Type mismatch: expected value of type array or multiset, but got the value of type string (in line 22, at column 8)</expected-error>
+ <output-dir compare="Text">q06</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="global-aggregate">
@@ -5959,8 +5958,7 @@
</test-case>
<test-case FilePath="group-by">
<compilation-unit name="sugar-01-negative">
- <output-dir compare="Text">core-01</output-dir>
- <expected-error>ASX0037: Type mismatch: expected value of type array or multiset, but got the value of type bigint (in line 26, at column 26)</expected-error>
+ <output-dir compare="Text">sugar-01-negative</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="group-by">
@@ -7184,8 +7182,7 @@
</test-case>
<test-case FilePath="misc">
<compilation-unit name="query-ASTERIXDB-2550">
- <output-dir compare="Text">none</output-dir>
- <expected-error>ASX0037: Type mismatch: expected value of type array or multiset, but got the value of type object (in line 28, at column 2)</expected-error>
+ <output-dir compare="Text">query-ASTERIXDB-2550</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="misc">
@@ -8676,6 +8673,13 @@
</compilation-unit>
</test-case>
</test-group>
+ <test-group name="array-index/index-bad-fields">
+ <test-case FilePath="array-index">
+ <compilation-unit name="index-bad-fields">
+ <output-dir compare="Text">index-bad-fields</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
</test-group>
<test-group name="nestrecords">
<test-case FilePath="nestrecords">
@@ -11363,6 +11367,22 @@
<output-dir compare="Text">substring-after-5</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="string" check-warnings="true">
+ <compilation-unit name="invalid-unicode">
+ <output-dir compare="Text">invalid-unicode</output-dir>
+ <expected-warn>Function 'string-length' failed to evaluate because: Decoding error - got a low surrogate without a leading high surrogate</expected-warn>
+ <expected-warn>Function 'string-to-codepoint' failed to evaluate because: Decoding error - got a low surrogate without a leading high surrogate</expected-warn>
+ <expected-warn>Function 'trim' failed to evaluate because: Decoding error - got a low surrogate without a leading high surrogate</expected-warn>
+ <expected-warn>Function 'trim' failed to evaluate because: Decoding error - got a low surrogate without a leading high surrogate</expected-warn>
+ <expected-warn>Function 'rtrim' failed to evaluate because: Decoding error - got a low surrogate without a leading high surrogate</expected-warn>
+ <expected-warn>Function 'rtrim' failed to evaluate because: Decoding error - got a low surrogate without a leading high surrogate</expected-warn>
+ <expected-warn>Function 'ltrim' failed to evaluate because: Decoding error - got a low surrogate without a leading high surrogate</expected-warn>
+ <expected-warn>Function 'ltrim' failed to evaluate because: Decoding error - got a low surrogate without a leading high surrogate</expected-warn>
+ <expected-warn>Function 'reverse' failed to evaluate because: Decoding error - got a low surrogate without a leading high surrogate</expected-warn>
+ <expected-warn>Function 'position' failed to evaluate because: Decoding error - got a low surrogate without a leading high surrogate</expected-warn>
+ <expected-warn>Function 'position1' failed to evaluate because: Decoding error - got a low surrogate without a leading high surrogate</expected-warn>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="subquery">
<test-case FilePath="subquery">
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index b0826e8..4910343 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -87,6 +87,7 @@
PARQUET_CONTAINS_OVERFLOWED_BIGINT(57),
UNEXPECTED_ERROR_ENCOUNTERED(58),
INVALID_PARQUET_FILE(59),
+ FUNCTION_EVALUATION_FAILED(60),
UNSUPPORTED_JRE(100),
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 0bf523a..074245c 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -94,6 +94,7 @@
57 = Parquet file(s) contain unsigned integer that is larger than the '%1$s' range
58 = Error encountered: %1$s
59 = Invalid Parquet file: %1$s. Reason: %2$s
+60 = Function '%1$s' failed to evaluate because: %2$s
100 = Unsupported JRE: %1$s
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
index 1abf300..498630a 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
@@ -35,6 +35,7 @@
import org.apache.asterix.om.types.AbstractCollectionType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.asterix.om.utils.RecordUtil;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -240,8 +241,9 @@
* Traverse each distinct record path and invoke the appropriate commands for each scenario. Here, we keep track
* of the record/list type at each step and give this to each command.
*/
- public static void walkArrayPath(Index index, ARecordType baseRecordType, List<String> flattenedFieldName,
- List<Boolean> unnestFlags, TypeTrackerCommandExecutor commandExecutor) throws AlgebricksException {
+ public static void walkArrayPath(Index index, Index.ArrayIndexElement workingElement, ARecordType baseRecordType,
+ List<String> flattenedFieldName, List<Boolean> unnestFlags, TypeTrackerCommandExecutor commandExecutor)
+ throws AlgebricksException {
ArrayPath arrayPath = new ArrayPath(flattenedFieldName, unnestFlags).invoke();
List<List<String>> fieldNamesPerArray = arrayPath.fieldNamesPerArray;
List<Boolean> unnestFlagsPerArray = arrayPath.unnestFlagsPerArray;
@@ -253,7 +255,8 @@
IAType workingType = baseRecordType;
for (int i = 0; i < fieldNamesPerArray.size(); i++) {
- ARecordType startingStepRecordType = (isTrackingType) ? (ARecordType) workingType : null;
+ ARecordType startingStepRecordType =
+ (isTrackingType) ? (ARecordType) workingType : RecordUtil.FULLY_OPEN_RECORD_TYPE;
if (isTrackingType) {
if (!workingType.getTypeTag().equals(ATypeTag.OBJECT)) {
throw new AsterixException(ErrorCode.COMPILATION_ERROR, "Mismatched record type to depth-"
@@ -286,15 +289,15 @@
}
}
boolean isFirstArrayStep = i == 0;
- boolean isLastUnnestInIntermediateStep = i < fieldNamesPerArray.size() - 1;
+ boolean isLastUnnestInIntermediateStep = i <= fieldNamesPerArray.size() - 1;
commandExecutor.executeActionOnEachArrayStep(startingStepRecordType, workingType,
fieldNamesPerArray.get(i), isFirstArrayStep, isLastUnnestInIntermediateStep);
}
if (i == fieldNamesPerArray.size() - 1) {
boolean isNonArrayStep = !unnestFlagsPerArray.get(i);
- commandExecutor.executeActionOnFinalArrayStep(startingStepRecordType, fieldNamesPerArray.get(i),
- isNonArrayStep, requiresOnlyOneUnnest);
+ commandExecutor.executeActionOnFinalArrayStep(workingElement, baseRecordType, startingStepRecordType,
+ fieldNamesPerArray.get(i), isNonArrayStep, requiresOnlyOneUnnest);
}
}
}
@@ -341,8 +344,9 @@
List<String> fieldName, boolean isFirstArrayStep, boolean isLastUnnestInIntermediateStep)
throws AlgebricksException;
- void executeActionOnFinalArrayStep(ARecordType startingStepRecordType, List<String> fieldName,
- boolean isNonArrayStep, boolean requiresOnlyOneUnnest) throws AlgebricksException;
+ void executeActionOnFinalArrayStep(Index.ArrayIndexElement workingElement, ARecordType baseRecordType,
+ ARecordType startingStepRecordType, List<String> fieldName, boolean isNonArrayStep,
+ boolean requiresOnlyOneUnnest) throws AlgebricksException;
}
private static class ArrayPath {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
index c8a7ee1..de0ca5a 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
@@ -158,7 +158,7 @@
int flattenedListPos = 0;
for (Index.ArrayIndexElement e : arrayIndexDetails.getElementList()) {
for (int i = 0; i < e.getProjectList().size(); i++) {
- addSKEvalFactories(isOverridingKeyFieldTypes ? enforcedItemType : itemType, flattenedListPos, false);
+ addSKEvalFactories(itemType, flattenedListPos, false, e);
Pair<IAType, Boolean> keyTypePair = ArrayIndexUtil.getNonNullableOpenFieldType(e.getTypeList().get(i),
e.getUnnestList(), e.getProjectList().get(i), itemType);
IAType keyType = keyTypePair.first;
@@ -203,7 +203,7 @@
ARecordType filterItemType =
((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterSourceIndicator() == 0 ? itemType
: metaType;
- addSKEvalFactories(itemType, numSecondaryKeys, true);
+ addSKEvalFactories(itemType, numSecondaryKeys, true, null);
Pair<IAType, Boolean> keyTypePair;
keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, filterItemType);
IAType type = keyTypePair.first;
@@ -230,8 +230,8 @@
return fieldPermutation;
}
- protected void addSKEvalFactories(ARecordType recordType, int fieldPos, boolean isFilterField)
- throws AlgebricksException {
+ protected void addSKEvalFactories(ARecordType recordType, int fieldPos, boolean isFilterField,
+ Index.ArrayIndexElement workingElement) throws AlgebricksException {
if (isFilterField) {
addFilterFieldToBuilder(recordType);
return;
@@ -245,7 +245,8 @@
} else {
EvalFactoryAndRecDescInvoker commandExecutor =
new EvalFactoryAndRecDescInvoker(!evalFactoryAndRecDescStackBuilder.isUnnestEvalPopulated());
- ArrayIndexUtil.walkArrayPath(index, recordType, flattenedFieldName, workingUnnestFlags, commandExecutor);
+ ArrayIndexUtil.walkArrayPath(index, workingElement, recordType, flattenedFieldName, workingUnnestFlags,
+ commandExecutor);
}
}
@@ -267,12 +268,6 @@
spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
sourceOp = targetOp;
- if (arrayIndexDetails.isOverridingKeyFieldTypes() && !enforcedItemType.equals(itemType)) {
- // If we have an enforced type, insert a "cast" after the primary index scan.
- targetOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced());
- spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
- sourceOp = targetOp;
- }
// We do not index meta fields. Project away meta fields if they exist.
if (dataset.hasMetaPart()) {
@@ -305,7 +300,15 @@
if (anySecondaryKeyIsNullable || arrayIndexDetails.isOverridingKeyFieldTypes()) {
// If any of the secondary fields are nullable, then we need to filter out the nulls.
- targetOp = createFilterAnyUnknownSelectOp(spec, numTotalSecondaryKeys, secondaryRecDesc);
+ List<IAType> secondaryKeyTypes = new ArrayList<>();
+ if (arrayIndexDetails.isOverridingKeyFieldTypes() && !enforcedItemType.equals(itemType)) {
+ for (Index.ArrayIndexElement arrayIndexElement : arrayIndexDetails.getElementList()) {
+ List<IAType> typeList = arrayIndexElement.getTypeList();
+ secondaryKeyTypes.addAll(typeList);
+ }
+ }
+ targetOp = createCastFilterAnyUnknownSelectOp(spec, numTotalSecondaryKeys, secondaryRecDesc,
+ secondaryKeyTypes);
spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
sourceOp = targetOp;
}
@@ -556,8 +559,9 @@
}
@Override
- public void executeActionOnFinalArrayStep(ARecordType startingStepRecordType, List<String> fieldName,
- boolean isNonArrayStep, boolean requiresOnlyOneUnnest) throws AlgebricksException {
+ public void executeActionOnFinalArrayStep(Index.ArrayIndexElement workingElement, ARecordType baseRecordType,
+ ARecordType startingStepRecordType, List<String> fieldName, boolean isNonArrayStep,
+ boolean requiresOnlyOneUnnest) throws AlgebricksException {
// If the final value is nested inside a record, add this SEF.
if (!isNonArrayStep) {
return;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index 694b153..ab64b18 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -478,9 +478,22 @@
return createFilterSelectOp(spec, numSecondaryKeyFields, secondaryRecDesc, AndDescriptor::new);
}
+ public AlgebricksMetaOperatorDescriptor createCastFilterAnyUnknownSelectOp(JobSpecification spec,
+ int numSecondaryKeyFields, RecordDescriptor secondaryRecDesc, List<IAType> castFieldTypes)
+ throws AlgebricksException {
+ return createFilterSelectOp(spec, numSecondaryKeyFields, secondaryRecDesc, AndDescriptor::new, castFieldTypes);
+ }
+
private AlgebricksMetaOperatorDescriptor createFilterSelectOp(JobSpecification spec, int numSecondaryKeyFields,
RecordDescriptor secondaryRecDesc, Supplier<AbstractFunctionDescriptor> predicatesCombinerFuncSupplier)
throws AlgebricksException {
+ return createFilterSelectOp(spec, numSecondaryKeyFields, secondaryRecDesc, predicatesCombinerFuncSupplier,
+ Collections.emptyList());
+ }
+
+ private AlgebricksMetaOperatorDescriptor createFilterSelectOp(JobSpecification spec, int numSecondaryKeyFields,
+ RecordDescriptor secondaryRecDesc, Supplier<AbstractFunctionDescriptor> predicatesCombinerFuncSupplier,
+ List<IAType> castFieldTypes) throws AlgebricksException {
IScalarEvaluatorFactory[] predicateArgsEvalFactories = new IScalarEvaluatorFactory[numSecondaryKeyFields];
NotDescriptor notDesc = new NotDescriptor();
notDesc.setSourceLocation(sourceLoc);
@@ -489,8 +502,14 @@
for (int i = 0; i < numSecondaryKeyFields; i++) {
// Access column i, and apply 'is not null'.
ColumnAccessEvalFactory columnAccessEvalFactory = new ColumnAccessEvalFactory(i);
+ IScalarEvaluatorFactory evalFactory = columnAccessEvalFactory;
+ if (castFieldTypes != null && !castFieldTypes.isEmpty()) {
+ IScalarEvaluatorFactory[] castArg = new IScalarEvaluatorFactory[] { columnAccessEvalFactory };
+ evalFactory = createCastFunction(castFieldTypes.get(i), BuiltinType.ANY, index.isEnforced(), sourceLoc)
+ .createEvaluatorFactory(castArg);
+ }
IScalarEvaluatorFactory isUnknownEvalFactory =
- isUnknownDesc.createEvaluatorFactory(new IScalarEvaluatorFactory[] { columnAccessEvalFactory });
+ isUnknownDesc.createEvaluatorFactory(new IScalarEvaluatorFactory[] { evalFactory });
IScalarEvaluatorFactory notEvalFactory =
notDesc.createEvaluatorFactory(new IScalarEvaluatorFactory[] { isUnknownEvalFactory });
predicateArgsEvalFactories[i] = notEvalFactory;
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/exceptions/ExceptionUtil.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/exceptions/ExceptionUtil.java
index ccb3a8d..daa9d83 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/exceptions/ExceptionUtil.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/exceptions/ExceptionUtil.java
@@ -19,6 +19,10 @@
package org.apache.asterix.om.exceptions;
+import static org.apache.asterix.common.exceptions.ErrorCode.FUNCTION_EVALUATION_FAILED;
+import static org.apache.hyracks.api.exceptions.ErrorCode.INVALID_STRING_UNICODE;
+import static org.apache.hyracks.api.util.ExceptionUtils.isErrorCode;
+
import java.util.function.Supplier;
import org.apache.asterix.common.exceptions.ErrorCode;
@@ -26,6 +30,7 @@
import org.apache.asterix.om.types.EnumDeserializer;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.api.exceptions.Warning;
@@ -143,6 +148,13 @@
warnInvalidValue(ctx, srcLoc, fid, argIdx, argValue, ErrorCode.NEGATIVE_VALUE);
}
+ public static void warnFunctionEvalFailed(IEvaluatorContext ctx, SourceLocation srcLoc, FunctionIdentifier fid,
+ String errMsg) {
+ if (ctx.getWarningCollector().shouldWarn()) {
+ ctx.getWarningCollector().warn(Warning.of(srcLoc, FUNCTION_EVALUATION_FAILED, fid.getName(), errMsg));
+ }
+ }
+
private static void warnInvalidValue(IEvaluatorContext ctx, SourceLocation srcLoc, FunctionIdentifier fid,
int argIdx, double argValue, ErrorCode errorCode) {
IWarningCollector warningCollector = ctx.getWarningCollector();
@@ -151,4 +163,8 @@
Warning.of(srcLoc, errorCode, fid.getName(), indexToPosition(argIdx), Double.toString(argValue)));
}
}
+
+ public static boolean isStringUnicodeError(HyracksDataException throwable) {
+ return isErrorCode(throwable, INVALID_STRING_UNICODE);
+ }
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/scalar/AbstractScalarAggregateDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/scalar/AbstractScalarAggregateDescriptor.java
index 064c861..2fd065d 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/scalar/AbstractScalarAggregateDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/scalar/AbstractScalarAggregateDescriptor.java
@@ -80,7 +80,7 @@
public IScalarEvaluator createScalarEvaluator(IEvaluatorContext ctx) throws HyracksDataException {
// Use ScanCollection to iterate over list items.
ScanCollectionUnnestingFunctionFactory scanCollectionFactory =
- new ScanCollectionUnnestingFunctionFactory(args[0], sourceLoc);
+ new ScanCollectionUnnestingFunctionFactory(args[0], sourceLoc, getIdentifier());
return createScalarAggregateEvaluator(aggFuncFactory.createAggregateEvaluator(ctx),
scanCollectionFactory, ctx);
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AbstractBinaryStringEval.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AbstractBinaryStringEval.java
index 2fc8654..204020e 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AbstractBinaryStringEval.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AbstractBinaryStringEval.java
@@ -106,6 +106,13 @@
// The actual processing.
try {
process(leftStringPointable, rightStringPointable, resultPointable);
+ } catch (HyracksDataException ex) {
+ if (ExceptionUtil.isStringUnicodeError(ex)) {
+ PointableHelper.setNull(resultPointable);
+ ExceptionUtil.warnFunctionEvalFailed(ctx, sourceLoc, funcID, ex.getMessageNoCode());
+ return;
+ }
+ throw ex;
} catch (IOException e) {
throw HyracksDataException.create(e);
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AbstractUnaryStringStringEval.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AbstractUnaryStringStringEval.java
index 5efe529..d92c9e9 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AbstractUnaryStringStringEval.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/AbstractUnaryStringStringEval.java
@@ -84,6 +84,13 @@
try {
process(stringPtr, resultPointable);
writeResult(resultPointable);
+ } catch (HyracksDataException ex) {
+ if (ExceptionUtil.isStringUnicodeError(ex)) {
+ PointableHelper.setNull(resultPointable);
+ ExceptionUtil.warnFunctionEvalFailed(ctx, sourceLoc, funcID, ex.getMessageNoCode());
+ return;
+ }
+ throw ex;
} catch (IOException e) {
throw HyracksDataException.create(e);
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringContainsDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringContainsDescriptor.java
index eeec70f..f6b2a2d 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringContainsDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringContainsDescriptor.java
@@ -46,7 +46,8 @@
return new AbstractBinaryStringBoolEval(ctx, args[0], args[1], BuiltinFunctions.STRING_CONTAINS,
sourceLoc) {
@Override
- protected boolean compute(UTF8StringPointable left, UTF8StringPointable right) {
+ protected boolean compute(UTF8StringPointable left, UTF8StringPointable right)
+ throws HyracksDataException {
return UTF8StringPointable.contains(left, right, false);
}
};
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringLengthDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringLengthDescriptor.java
index 47caf14..da23ae5 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringLengthDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringLengthDescriptor.java
@@ -19,7 +19,6 @@
package org.apache.asterix.runtime.evaluators.functions;
import java.io.DataOutput;
-import java.io.IOException;
import org.apache.asterix.common.annotations.MissingNullInOutFunction;
import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
@@ -56,13 +55,13 @@
@Override
public IScalarEvaluator createScalarEvaluator(final IEvaluatorContext ctx) throws HyracksDataException {
return new IScalarEvaluator() {
- private AMutableInt64 result = new AMutableInt64(0);
- private ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
- private DataOutput out = resultStorage.getDataOutput();
- private IPointable inputArg = new VoidPointable();
- private IScalarEvaluator eval = args[0].createScalarEvaluator(ctx);
+ private final AMutableInt64 result = new AMutableInt64(0);
+ private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
+ private final DataOutput out = resultStorage.getDataOutput();
+ private final IPointable inputArg = new VoidPointable();
+ private final IScalarEvaluator eval = args[0].createScalarEvaluator(ctx);
@SuppressWarnings("unchecked")
- private ISerializerDeserializer<AInt64> int64Serde =
+ private final ISerializerDeserializer<AInt64> int64Serde =
SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT64);
@Override
@@ -89,8 +88,14 @@
result.setValue(len);
int64Serde.serialize(result, out);
resultPointable.set(resultStorage);
- } catch (IOException e1) {
- throw HyracksDataException.create(e1);
+ } catch (HyracksDataException ex) {
+ if (ExceptionUtil.isStringUnicodeError(ex)) {
+ PointableHelper.setNull(resultPointable);
+ ExceptionUtil.warnFunctionEvalFailed(ctx, sourceLoc, getIdentifier(),
+ ex.getMessageNoCode());
+ return;
+ }
+ throw ex;
}
}
};
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringPositionDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringPositionDescriptor.java
index 6c06056..051083f 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringPositionDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringPositionDescriptor.java
@@ -47,7 +47,8 @@
StringPositionDescriptor.this.getIdentifier(), sourceLoc) {
@Override
- protected int compute(UTF8StringPointable left, UTF8StringPointable right) {
+ protected int compute(UTF8StringPointable left, UTF8StringPointable right)
+ throws HyracksDataException {
return UTF8StringPointable.findInCodePoint(left, right, false);
}
};
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringPositionOffset1Descriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringPositionOffset1Descriptor.java
index 93ada0f..668e03c 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringPositionOffset1Descriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringPositionOffset1Descriptor.java
@@ -47,7 +47,8 @@
StringPositionOffset1Descriptor.this.getIdentifier(), sourceLoc) {
@Override
- protected int compute(UTF8StringPointable left, UTF8StringPointable right) {
+ protected int compute(UTF8StringPointable left, UTF8StringPointable right)
+ throws HyracksDataException {
int pos = UTF8StringPointable.findInCodePoint(left, right, false);
return pos < 0 ? pos : pos + 1;
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringToCodePointDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringToCodePointDescriptor.java
index 2f6a223..3320c2e 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringToCodePointDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringToCodePointDescriptor.java
@@ -19,7 +19,6 @@
package org.apache.asterix.runtime.evaluators.functions;
import java.io.DataOutput;
-import java.io.IOException;
import org.apache.asterix.builders.OrderedListBuilder;
import org.apache.asterix.common.annotations.MissingNullInOutFunction;
@@ -60,14 +59,14 @@
@Override
public IScalarEvaluator createScalarEvaluator(final IEvaluatorContext ctx) throws HyracksDataException {
return new IScalarEvaluator() {
- protected final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
- protected final DataOutput out = resultStorage.getDataOutput();
- protected final IPointable argPtr = new VoidPointable();
- protected final IScalarEvaluator stringEval = args[0].createScalarEvaluator(ctx);
- protected final AOrderedListType intListType = new AOrderedListType(BuiltinType.AINT64, null);
+ private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
+ private final DataOutput out = resultStorage.getDataOutput();
+ private final IPointable argPtr = new VoidPointable();
+ private final IScalarEvaluator stringEval = args[0].createScalarEvaluator(ctx);
+ private final AOrderedListType intListType = new AOrderedListType(BuiltinType.AINT64, null);
- private OrderedListBuilder listBuilder = new OrderedListBuilder();
- private ArrayBackedValueStorage inputVal = new ArrayBackedValueStorage();
+ private final OrderedListBuilder listBuilder = new OrderedListBuilder();
+ private final ArrayBackedValueStorage inputVal = new ArrayBackedValueStorage();
@SuppressWarnings("unchecked")
private final ISerializerDeserializer<AInt64> int64Serde =
@@ -109,8 +108,14 @@
}
listBuilder.write(out, true);
result.set(resultStorage);
- } catch (IOException e1) {
- throw HyracksDataException.create(e1);
+ } catch (HyracksDataException ex) {
+ if (ExceptionUtil.isStringUnicodeError(ex)) {
+ PointableHelper.setNull(result);
+ ExceptionUtil.warnFunctionEvalFailed(ctx, sourceLoc, getIdentifier(),
+ ex.getMessageNoCode());
+ return;
+ }
+ throw ex;
}
}
};
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/utils/StringTrimmer.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/utils/StringTrimmer.java
index 8dc41f5..0ddf459 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/utils/StringTrimmer.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/utils/StringTrimmer.java
@@ -22,6 +22,7 @@
import java.io.IOException;
import org.apache.asterix.runtime.evaluators.functions.StringEvaluatorUtils;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IPointable;
import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
import org.apache.hyracks.data.std.util.ByteArrayAccessibleOutputStream;
@@ -51,7 +52,7 @@
* @param resultArray
* , the byte array to hold results.
*/
- public StringTrimmer(UTF8StringBuilder resultBuilder, GrowableArray resultArray) {
+ public StringTrimmer(UTF8StringBuilder resultBuilder, GrowableArray resultArray) throws HyracksDataException {
this(resultBuilder, resultArray, null);
}
@@ -63,7 +64,8 @@
* @param pattern
* , the string that is used to construct the charset for trimming.
*/
- public StringTrimmer(UTF8StringBuilder resultBuilder, GrowableArray resultArray, UTF8StringPointable pattern) {
+ public StringTrimmer(UTF8StringBuilder resultBuilder, GrowableArray resultArray, UTF8StringPointable pattern)
+ throws HyracksDataException {
this.resultBuilder = resultBuilder;
this.resultArray = resultArray;
if (pattern != null) {
@@ -78,7 +80,7 @@
* @param patternPtr
* , a pattern string.
*/
- public void build(UTF8StringPointable patternPtr) {
+ public void build(UTF8StringPointable patternPtr) throws HyracksDataException {
final boolean newPattern = (codePointSet.size() == 0) || lastPatternPtr.compareTo(patternPtr) != 0;
if (newPattern) {
StringEvaluatorUtils.copyResetUTF8Pointable(patternPtr, lastPatternStorage, lastPatternPtr);
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/unnestingfunctions/std/ScanCollectionDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/unnestingfunctions/std/ScanCollectionDescriptor.java
index ff6dd89..6f9dd5c 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/unnestingfunctions/std/ScanCollectionDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/unnestingfunctions/std/ScanCollectionDescriptor.java
@@ -21,11 +21,11 @@
import java.io.IOException;
+import org.apache.asterix.om.exceptions.ExceptionUtil;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.runtime.evaluators.common.ListAccessor;
-import org.apache.asterix.runtime.exceptions.TypeMismatchException;
import org.apache.asterix.runtime.unnestingfunctions.base.AbstractUnnestingFunctionDynamicDescriptor;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
@@ -52,7 +52,7 @@
@Override
public IUnnestingEvaluatorFactory createUnnestingEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new ScanCollectionUnnestingFunctionFactory(args[0], sourceLoc);
+ return new ScanCollectionUnnestingFunctionFactory(args[0], sourceLoc, getIdentifier());
}
public static class ScanCollectionUnnestingFunctionFactory implements IUnnestingEvaluatorFactory {
@@ -60,10 +60,13 @@
private static final long serialVersionUID = 1L;
private IScalarEvaluatorFactory listEvalFactory;
private final SourceLocation sourceLoc;
+ private final FunctionIdentifier funID;
- public ScanCollectionUnnestingFunctionFactory(IScalarEvaluatorFactory arg, SourceLocation sourceLoc) {
+ public ScanCollectionUnnestingFunctionFactory(IScalarEvaluatorFactory arg, SourceLocation sourceLoc,
+ FunctionIdentifier funID) {
this.listEvalFactory = arg;
this.sourceLoc = sourceLoc;
+ this.funID = funID;
}
@Override
@@ -88,8 +91,9 @@
}
if (typeTag != ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
&& typeTag != ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
- throw new TypeMismatchException(sourceLoc, typeTag, ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG,
- ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
+ ExceptionUtil.warnTypeMismatch(ctx, sourceLoc, funID, typeTag, 0, ATypeTag.MULTISET);
+ metUnknown = true;
+ return;
}
listAccessor.reset(inputVal.getByteArray(), inputVal.getStartOffset());
itemIndex = 0;
diff --git a/hyracks-fullstack/algebricks/algebricks-data/pom.xml b/hyracks-fullstack/algebricks/algebricks-data/pom.xml
index 81edbda..8ca0765 100644
--- a/hyracks-fullstack/algebricks/algebricks-data/pom.xml
+++ b/hyracks-fullstack/algebricks/algebricks-data/pom.xml
@@ -54,11 +54,6 @@
</dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-api</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/hyracks-fullstack/hyracks/hyracks-api/pom.xml b/hyracks-fullstack/hyracks/hyracks-api/pom.xml
index 047f066..131731d 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-api/pom.xml
@@ -65,6 +65,13 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
index 7291473..8170f07 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
@@ -154,6 +154,7 @@
PARSING_ERROR(124),
INVALID_INVERTED_LIST_TYPE_TRAITS(125),
ILLEGAL_STATE(126),
+ INVALID_STRING_UNICODE(127),
// Compilation error codes.
RULECOLLECTION_NOT_INSTANCE_OF_LIST(10000),
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksException.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksException.java
index 977e5d2..12f1095 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksException.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/HyracksException.java
@@ -158,6 +158,10 @@
return message;
}
+ public String getMessageNoCode() {
+ return ErrorMessageUtil.getMessageNoCode(component, getMessage());
+ }
+
@Override
public String toString() {
return getLocalizedMessage();
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ErrorMessageUtil.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ErrorMessageUtil.java
index 70b13fa..cb0d579 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ErrorMessageUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ErrorMessageUtil.java
@@ -125,6 +125,13 @@
}
}
+ public static String getMessageNoCode(String component, String message) {
+ if (NONE.equals(component)) {
+ return message;
+ }
+ return message.substring(message.indexOf(":") + 2);
+ }
+
public static String getCauseMessage(Throwable t) {
if (t instanceof IFormattedException) {
return t.getMessage();
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExceptionUtils.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExceptionUtils.java
index 7147542..e07cdd4 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExceptionUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExceptionUtils.java
@@ -207,4 +207,16 @@
public static String getMessageOrToString(Throwable e) {
return e instanceof IFormattedException ? e.getMessage() : e.toString();
}
+
+ /**
+ * Checks if the error code of the throwable is of the provided type
+ *
+ * @param throwable throwable with error code
+ * @param code error code to match against
+ *
+ * @return true if error code matches, false otherwise
+ */
+ public static boolean isErrorCode(HyracksDataException throwable, ErrorCode code) {
+ return throwable.getError().isPresent() && throwable.getError().get() == code;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringReader.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/util/string/UTF8StringReader.java
similarity index 100%
rename from hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringReader.java
rename to hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/util/string/UTF8StringReader.java
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
similarity index 96%
rename from hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
rename to hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
index c0475b1..a50cc31 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
@@ -17,6 +17,8 @@
package org.apache.hyracks.util.string;
+import static org.apache.hyracks.api.exceptions.ErrorCode.INVALID_STRING_UNICODE;
+
import java.io.ByteArrayOutputStream;
import java.io.DataInput;
import java.io.DataOutput;
@@ -27,6 +29,7 @@
import java.io.UTFDataFormatException;
import java.lang.ref.SoftReference;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.util.encoding.VarLenIntEncoderDecoder;
/**
@@ -35,10 +38,11 @@
*/
public class UTF8StringUtil {
+ public static final String MALFORMED_BYTES = "malformed bytes";
public static final String LOW_SURROGATE_WITHOUT_HIGH_SURROGATE =
- "Decoding error: got a low surrogate without a leading high surrogate";
+ "got a low surrogate without a leading high surrogate";
public static final String HIGH_SURROGATE_WITHOUT_LOW_SURROGATE =
- "Decoding error: got a high surrogate without a following low surrogate";
+ "got a high surrogate without a following low surrogate";
private UTF8StringUtil() {
}
@@ -96,12 +100,12 @@
}
}
- public static int codePointAt(byte[] b, int s) {
+ public static int codePointAt(byte[] b, int s) throws HyracksDataException {
char c1 = charAt(b, s);
if (Character.isLowSurrogate(c1)) {
// In this case, the index s doesn't point to a correct position
- throw new IllegalArgumentException(LOW_SURROGATE_WITHOUT_HIGH_SURROGATE);
+ throw HyracksDataException.create(INVALID_STRING_UNICODE, LOW_SURROGATE_WITHOUT_HIGH_SURROGATE);
}
if (Character.isHighSurrogate(c1)) {
@@ -112,19 +116,19 @@
if (Character.isLowSurrogate(c2)) {
return Character.toCodePoint(c1, c2);
} else {
- throw new IllegalArgumentException(HIGH_SURROGATE_WITHOUT_LOW_SURROGATE);
+ throw HyracksDataException.create(INVALID_STRING_UNICODE, HIGH_SURROGATE_WITHOUT_LOW_SURROGATE);
}
}
return c1;
}
- public static int codePointSize(byte[] b, int s) {
+ public static int codePointSize(byte[] b, int s) throws HyracksDataException {
char c1 = charAt(b, s);
int size1 = charSize(b, s);
if (Character.isLowSurrogate(c1)) {
- throw new IllegalArgumentException(LOW_SURROGATE_WITHOUT_HIGH_SURROGATE);
+ throw HyracksDataException.create(INVALID_STRING_UNICODE, LOW_SURROGATE_WITHOUT_HIGH_SURROGATE);
}
if (Character.isHighSurrogate(c1)) {
@@ -204,7 +208,7 @@
return charCount;
}
- public static int getNumCodePoint(byte[] b, int s) {
+ public static int getNumCodePoint(byte[] b, int s) throws HyracksDataException {
int len = getUTFLength(b, s);
int pos = s + getNumBytesToStoreLength(len);
int end = pos + len;
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
similarity index 100%
rename from hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
rename to hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
index 4d9c60b..7db5d49 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
@@ -144,6 +144,8 @@
124 = Parsing error %s: %s
125 = Invalid inverted list type traits: %1$s
126 = Illegal state. %1$s
+127 = Decoding error - %1$s
+
10000 = The given rule collection %1$s is not an instance of the List class.
10001 = Cannot compose partition constraint %1$s with %2$s
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringReaderWriterTest.java b/hyracks-fullstack/hyracks/hyracks-api/src/test/java/org/apache/hyracks/api/string/UTF8StringReaderWriterTest.java
similarity index 95%
rename from hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringReaderWriterTest.java
rename to hyracks-fullstack/hyracks/hyracks-api/src/test/java/org/apache/hyracks/api/string/UTF8StringReaderWriterTest.java
index 9010c9c..abba958 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringReaderWriterTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/test/java/org/apache/hyracks/api/string/UTF8StringReaderWriterTest.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.hyracks.util.string;
+package org.apache.hyracks.api.string;
import static org.apache.hyracks.util.string.UTF8StringSample.EMPTY_STRING;
import static org.apache.hyracks.util.string.UTF8StringSample.STRING_LEN_127;
@@ -37,6 +37,8 @@
import java.io.DataOutputStream;
import java.io.IOException;
+import org.apache.hyracks.util.string.UTF8StringReader;
+import org.apache.hyracks.util.string.UTF8StringWriter;
import org.junit.Test;
public class UTF8StringReaderWriterTest {
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringUtilTest.java b/hyracks-fullstack/hyracks/hyracks-api/src/test/java/org/apache/hyracks/api/string/UTF8StringUtilTest.java
similarity index 96%
rename from hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringUtilTest.java
rename to hyracks-fullstack/hyracks/hyracks-api/src/test/java/org/apache/hyracks/api/string/UTF8StringUtilTest.java
index 4eb1fc3..37ab002 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/test/java/org/apache/hyracks/util/string/UTF8StringUtilTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/test/java/org/apache/hyracks/api/string/UTF8StringUtilTest.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.hyracks.util.string;
+package org.apache.hyracks.api.string;
import static org.apache.hyracks.util.string.UTF8StringSample.STRING_LEN_127;
import static org.apache.hyracks.util.string.UTF8StringSample.STRING_LEN_128;
@@ -47,6 +47,8 @@
import java.io.IOException;
import java.util.List;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.util.string.UTF8StringUtil;
import org.junit.Test;
public class UTF8StringUtilTest {
@@ -179,7 +181,7 @@
}
@Test
- public void testGetNumCodePoint() {
+ public void testGetNumCodePoint() throws HyracksDataException {
String str = "\uD83D\uDC69\u200D\uD83D\uDC69\u200D\uD83D\uDC67\u200D\uD83D\uDC66";
assertEquals(getNumCodePoint(writeStringToBytes(str), 0), 7);
diff --git a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/UTF8StringPointable.java b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/UTF8StringPointable.java
index 49f6221..47d2488 100644
--- a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/UTF8StringPointable.java
+++ b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/UTF8StringPointable.java
@@ -18,8 +18,10 @@
*/
package org.apache.hyracks.data.std.primitive;
+import static org.apache.hyracks.api.exceptions.ErrorCode.INVALID_STRING_UNICODE;
import static org.apache.hyracks.util.string.UTF8StringUtil.HIGH_SURROGATE_WITHOUT_LOW_SURROGATE;
import static org.apache.hyracks.util.string.UTF8StringUtil.LOW_SURROGATE_WITHOUT_HIGH_SURROGATE;
+import static org.apache.hyracks.util.string.UTF8StringUtil.MALFORMED_BYTES;
import java.io.IOException;
import java.nio.charset.Charset;
@@ -120,15 +122,15 @@
return UTF8StringUtil.charSize(bytes, start + offset);
}
- public int codePointAt(int offset) {
+ public int codePointAt(int offset) throws HyracksDataException {
return UTF8StringUtil.codePointAt(bytes, start + offset);
}
- public int codePointSize(int offset) {
+ public int codePointSize(int offset) throws HyracksDataException {
return UTF8StringUtil.codePointSize(bytes, start + offset);
}
- public void getCodePoints(IntCollection codePointSet) {
+ public void getCodePoints(IntCollection codePointSet) throws HyracksDataException {
int byteIdx = 0;
while (byteIdx < utf8Length) {
codePointSet.add(codePointAt(metaLength + byteIdx));
@@ -136,7 +138,7 @@
}
if (byteIdx != utf8Length) {
- throw new IllegalArgumentException("Decoding error: malformed bytes");
+ throw HyracksDataException.create(INVALID_STRING_UNICODE, MALFORMED_BYTES);
}
}
@@ -202,7 +204,7 @@
other.getStartOffset());
}
- public int find(UTF8StringPointable pattern, boolean ignoreCase) {
+ public int find(UTF8StringPointable pattern, boolean ignoreCase) throws HyracksDataException {
return find(this, pattern, ignoreCase);
}
@@ -227,7 +229,8 @@
* to ignore case or not.
* @return the byte offset of the first character of the matching string. Not including the MetaLength.
*/
- public static int find(UTF8StringPointable src, UTF8StringPointable pattern, boolean ignoreCase) {
+ public static int find(UTF8StringPointable src, UTF8StringPointable pattern, boolean ignoreCase)
+ throws HyracksDataException {
return find(src, pattern, ignoreCase, 0);
}
@@ -240,7 +243,8 @@
* to ignore case or not.
* @return the offset in the unit of code point of the first character of the matching string. Not including the MetaLength.
*/
- public static int findInCodePoint(UTF8StringPointable src, UTF8StringPointable pattern, boolean ignoreCase) {
+ public static int findInCodePoint(UTF8StringPointable src, UTF8StringPointable pattern, boolean ignoreCase)
+ throws HyracksDataException {
return findInByteOrCodePoint(src, pattern, ignoreCase, 0, false);
}
@@ -256,7 +260,8 @@
* @return the byte offset of the first character of the matching string after <code>startMatchPos}</code>.
* Not including the MetaLength.
*/
- public static int find(UTF8StringPointable src, UTF8StringPointable pattern, boolean ignoreCase, int startMatch) {
+ public static int find(UTF8StringPointable src, UTF8StringPointable pattern, boolean ignoreCase, int startMatch)
+ throws HyracksDataException {
return findInByteOrCodePoint(src, pattern, ignoreCase, startMatch, true);
}
@@ -272,13 +277,13 @@
* @return the offset in the unit of code point of the first character of the matching string. Not including the MetaLength.
*/
public static int findInCodePoint(UTF8StringPointable src, UTF8StringPointable pattern, boolean ignoreCase,
- int startMatch) {
+ int startMatch) throws HyracksDataException {
return findInByteOrCodePoint(src, pattern, ignoreCase, startMatch, false);
}
// If resultInByte is true, then return the position in bytes, otherwise return the position in code points
private static int findInByteOrCodePoint(UTF8StringPointable src, UTF8StringPointable pattern, boolean ignoreCase,
- int startMatch, boolean resultInByte) {
+ int startMatch, boolean resultInByte) throws HyracksDataException {
int startMatchPos = startMatch;
final int srcUtfLen = src.getUTF8Length();
final int pttnUtfLen = pattern.getUTF8Length();
@@ -317,7 +322,7 @@
return startMatchPos;
} else {
if (prevHighSurrogate) {
- throw new IllegalArgumentException(HIGH_SURROGATE_WITHOUT_LOW_SURROGATE);
+ throw HyracksDataException.create(INVALID_STRING_UNICODE, HIGH_SURROGATE_WITHOUT_LOW_SURROGATE);
}
return codePointCount;
}
@@ -333,7 +338,7 @@
codePointCount++;
prevHighSurrogate = false;
} else {
- throw new IllegalArgumentException(LOW_SURROGATE_WITHOUT_HIGH_SURROGATE);
+ throw HyracksDataException.create(INVALID_STRING_UNICODE, LOW_SURROGATE_WITHOUT_HIGH_SURROGATE);
}
} else {
codePointCount++;
@@ -345,11 +350,12 @@
return -1;
}
- public boolean contains(UTF8StringPointable pattern, boolean ignoreCase) {
+ public boolean contains(UTF8StringPointable pattern, boolean ignoreCase) throws HyracksDataException {
return contains(this, pattern, ignoreCase);
}
- public static boolean contains(UTF8StringPointable src, UTF8StringPointable pattern, boolean ignoreCase) {
+ public static boolean contains(UTF8StringPointable src, UTF8StringPointable pattern, boolean ignoreCase)
+ throws HyracksDataException {
return find(src, pattern, ignoreCase) >= 0;
}
@@ -678,9 +684,9 @@
endIndex = startIndex;
int cursorIndex = startIndex;
while (cursorIndex < srcUtfLen) {
- int codePioint = srcPtr.codePointAt(srcStart + cursorIndex);
+ int codePoint = srcPtr.codePointAt(srcStart + cursorIndex);
cursorIndex += srcPtr.codePointSize(srcStart + cursorIndex);
- if (!codePointSet.contains(codePioint)) {
+ if (!codePointSet.contains(codePoint)) {
endIndex = cursorIndex;
}
}
@@ -739,9 +745,9 @@
cursorIndex--;
if (UTF8StringUtil.isCharStart(srcPtr.bytes, cursorIndex)) {
ch = UTF8StringUtil.charAt(srcPtr.bytes, cursorIndex);
- if (Character.isHighSurrogate(ch) == false) {
- throw new IllegalArgumentException(
- "Decoding Error: no corresponding high surrogate found for the following low surrogate");
+ if (!Character.isHighSurrogate(ch)) {
+ throw HyracksDataException.create(INVALID_STRING_UNICODE,
+ LOW_SURROGATE_WITHOUT_HIGH_SURROGATE);
}
charSize += UTF8StringUtil.charSize(srcPtr.bytes, cursorIndex);
@@ -749,7 +755,7 @@
}
}
} else if (Character.isHighSurrogate(ch)) {
- throw new IllegalArgumentException("Decoding Error: get a high surrogate without low surrogate");
+ throw HyracksDataException.create(INVALID_STRING_UNICODE, HIGH_SURROGATE_WITHOUT_LOW_SURROGATE);
}
builder.appendUtf8StringPointable(srcPtr, cursorIndex, charSize);
diff --git a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/test/java/org/apache/hyracks/data/std/primitive/UTF8StringPointableTest.java b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/test/java/org/apache/hyracks/data/std/primitive/UTF8StringPointableTest.java
index f088c7e..45a5ba3 100644
--- a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/test/java/org/apache/hyracks/data/std/primitive/UTF8StringPointableTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/test/java/org/apache/hyracks/data/std/primitive/UTF8StringPointableTest.java
@@ -30,6 +30,7 @@
import java.io.IOException;
import java.util.Arrays;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.util.GrowableArray;
import org.apache.hyracks.data.std.util.UTF8StringBuilder;
import org.apache.hyracks.util.string.UTF8StringSample;
@@ -68,7 +69,7 @@
}
@Test
- public void testFindInCodePoint() {
+ public void testFindInCodePoint() throws HyracksDataException {
UTF8StringPointable strp = generateUTF8Pointable(STRING_EMOJI_FAMILY_OF_4 + EMOJI_BASKETBALL);
UTF8StringPointable pattern = generateUTF8Pointable(EMOJI_BASKETBALL);