[ASTERIXDB-2883][COMP] Improve null handling in UDF calls
- user model changes: no
- storage format changes: no
- interface changes: no
- IntroduceDynamicTypeCastForExternalFunctionRule
should not apply twice on the same operator
- Add null-call handling to ExternalTypeComputer
and ExternalScalarJavaFunctionEvaluator
Change-Id: I9c58e28f673606c5d0413bdc4cd3ba0c7c20eb8d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11306
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
Reviewed-by: Ian Maxon <imaxon@uci.edu>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java
index d3eb0c2..2c5a07d 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java
@@ -115,6 +115,13 @@
if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
return false;
}
- return op.acceptExpressionTransform(expr -> rewriteFunctionArgs(op, expr, context));
+ if (context.checkIfInDontApplySet(this, op)) {
+ return false;
+ }
+ boolean applied = op.acceptExpressionTransform(expr -> rewriteFunctionArgs(op, expr, context));
+ if (applied) {
+ context.addToDontApplySet(this, op);
+ }
+ return applied;
}
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.0.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.0.ddl.sqlpp
new file mode 100644
index 0000000..76cc70d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.0.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE externallibtest if exists;
+CREATE DATAVERSE externallibtest;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.1.lib.sqlpp
new file mode 100644
index 0000000..b2bf929
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.1.lib.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+install externallibtest testlib java admin admin target/data/externallib/asterix-external-data-testlib.zip
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.2.ddl.sqlpp
new file mode 100644
index 0000000..c7d6768
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.2.ddl.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use externallibtest;
+
+create function typeName(v) returns string
+ as "org.apache.asterix.external.library.TypeNameFactory" at testlib;
+
+create function typeNameNullCall(v) returns string
+ as "org.apache.asterix.external.library.TypeNameFactory" at testlib with {"null-call": true};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.3.query.sqlpp
new file mode 100644
index 0000000..2542d55
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.3.query.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use externallibtest;
+
+{
+ "missing": typeName(missing) is missing,
+ "missing_nullcall": typeNameNullCall(missing),
+ "null": typeName(null) is null,
+ "null_nullcall": typeNameNullCall(null),
+ "boolean": typeName(true),
+ "string": typeName("x")
+}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.4.ddl.sqlpp
new file mode 100644
index 0000000..2b27030
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_name/type_name.4.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE externallibtest;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/type_name/type_name.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/type_name/type_name.3.adm
new file mode 100644
index 0000000..a72c46c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/type_name/type_name.3.adm
@@ -0,0 +1 @@
+{ "missing": true, "missing_nullcall": "missing", "null": true, "null_nullcall": "null", "boolean": "boolean", "string": "string" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
index 28cbabb..6c67216 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
@@ -42,6 +42,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="external-library">
+ <compilation-unit name="type_name">
+ <output-dir compare="Text">type_name</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-library">
<compilation-unit name="type_validation">
<output-dir compare="Text">type_validation</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
index 33b0369..35d55f7 100755
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
@@ -19,7 +19,7 @@
package org.apache.asterix.external.library;
-import java.io.IOException;
+import static org.apache.asterix.om.types.EnumDeserializer.ATYPETAGDESERIALIZER;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
@@ -27,7 +27,9 @@
import org.apache.asterix.external.api.IExternalScalarFunction;
import org.apache.asterix.external.api.IFunctionFactory;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
+import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.evaluators.functions.PointableHelper;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -71,23 +73,36 @@
@Override
public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
try {
- setArguments(tuple);
- resultBuffer.reset();
- externalFunctionInstance.evaluate(functionHelper);
- if (!functionHelper.isValidResult()) {
- throw new RuntimeDataException(ErrorCode.EXTERNAL_UDF_RESULT_TYPE_ERROR);
+ boolean nullCall = finfo.getNullCall();
+ boolean hasNullArg = false;
+ for (int i = 0; i < argEvals.length; i++) {
+ argEvals[i].evaluate(tuple, inputVal);
+ if (!nullCall) {
+ byte[] inputValBytes = inputVal.getByteArray();
+ int inputValStartOffset = inputVal.getStartOffset();
+ ATypeTag typeTag = ATYPETAGDESERIALIZER.deserialize(inputValBytes[inputValStartOffset]);
+ if (typeTag == ATypeTag.MISSING) {
+ PointableHelper.setMissing(result);
+ return;
+ } else if (typeTag == ATypeTag.NULL) {
+ hasNullArg = true;
+ }
+ }
+ functionHelper.setArgument(i, inputVal);
}
- result.set(resultBuffer.getByteArray(), resultBuffer.getStartOffset(), resultBuffer.getLength());
- functionHelper.reset();
+ if (!nullCall && hasNullArg) {
+ PointableHelper.setNull(result);
+ } else {
+ resultBuffer.reset();
+ externalFunctionInstance.evaluate(functionHelper);
+ if (!functionHelper.isValidResult()) {
+ throw new RuntimeDataException(ErrorCode.EXTERNAL_UDF_RESULT_TYPE_ERROR);
+ }
+ result.set(resultBuffer.getByteArray(), resultBuffer.getStartOffset(), resultBuffer.getLength());
+ functionHelper.reset();
+ }
} catch (Exception e) {
throw HyracksDataException.create(e);
}
}
-
- public void setArguments(IFrameTupleReference tuple) throws IOException {
- for (int i = 0; i < argEvals.length; i++) {
- argEvals[i].evaluate(tuple, inputVal);
- functionHelper.setArgument(i, inputVal);
- }
- }
}
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFactory.java
new file mode 100644
index 0000000..335c383
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFactory.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.external.library;
+
+import org.apache.asterix.external.api.IExternalScalarFunction;
+import org.apache.asterix.external.api.IFunctionFactory;
+
+public class TypeNameFactory implements IFunctionFactory {
+ @Override
+ public IExternalScalarFunction getExternalFunction() {
+ return new TypeNameFunction();
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFunction.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFunction.java
new file mode 100644
index 0000000..1f03fcc
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeNameFunction.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.external.library;
+
+import org.apache.asterix.external.api.IExternalScalarFunction;
+import org.apache.asterix.external.api.IFunctionHelper;
+import org.apache.asterix.external.library.java.base.JString;
+
+public class TypeNameFunction implements IExternalScalarFunction {
+
+ private JString result;
+
+ @Override
+ public void deinitialize() {
+ // nothing to do here
+ }
+
+ @Override
+ public void evaluate(IFunctionHelper functionHelper) throws Exception {
+ String arg0TypeName = functionHelper.getArgument(0).getIAType().getTypeName();
+ result.setValue(arg0TypeName);
+ functionHelper.setResult(result);
+ }
+
+ @Override
+ public void initialize(IFunctionHelper functionHelper) {
+ result = (JString) functionHelper.getResultObject();
+ }
+}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
index 6751171..5fd96a6 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
@@ -69,7 +69,7 @@
IAType returnType = getType(function.getReturnType(), metadataProvider);
- IResultTypeComputer typeComputer = new ExternalTypeComputer(returnType, paramTypes);
+ IResultTypeComputer typeComputer = new ExternalTypeComputer(returnType, paramTypes, function.getNullCall());
ExternalFunctionLanguage lang = getExternalFunctionLanguage(function.getLanguage());
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalTypeComputer.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalTypeComputer.java
index 4a000a3..99d8ea2 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalTypeComputer.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalTypeComputer.java
@@ -33,8 +33,9 @@
public class ExternalTypeComputer extends AbstractResultTypeComputer {
- private IAType resultType;
- private List<IAType> paramPrimeTypes;
+ private final IAType resultType;
+ private final List<IAType> paramPrimeTypes;
+ private final boolean nullCall;
@Override
protected void checkArgType(FunctionIdentifier funcId, int argIndex, IAType type, SourceLocation sourceLoc)
@@ -47,14 +48,20 @@
}
}
- public ExternalTypeComputer(IAType resultPrimeType, List<IAType> paramPrimeTypes) {
+ public ExternalTypeComputer(IAType resultPrimeType, List<IAType> paramPrimeTypes, boolean nullCall) {
this.resultType = resultPrimeType.getTypeTag() == ATypeTag.ANY ? resultPrimeType
: AUnionType.createUnknownableType(resultPrimeType);
this.paramPrimeTypes = paramPrimeTypes;
+ this.nullCall = nullCall;
}
@Override
protected IAType getResultType(ILogicalExpression expr, IAType... strippedInputTypes) {
return resultType;
}
+
+ @Override
+ protected boolean propagateNullAndMissing() {
+ return !nullCall;
+ }
}