[NO ISSUE][COMP] Non-deterministic external functions
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add compiler support for non-deterministic external functions
- Constant fold only those external functions that are
declared as deterministic and are implemented in Java
(grammar default is non-deterministic)
- Add testcase
Change-Id: Iae7839cb7f6c21f8980867e31de78887a8e9e801
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/5205
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <imaxon@uci.edu>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index 176a678..d3093e7 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@ -43,10 +43,12 @@
import org.apache.asterix.formats.nontagged.TypeTraitProvider;
import org.apache.asterix.jobgen.QueryLogicalExpressionJobGen;
import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.om.base.ADouble;
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.constants.AsterixConstantValue;
import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
@@ -75,6 +77,7 @@
import org.apache.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
@@ -356,6 +359,12 @@
}
private boolean canConstantFold(ScalarFunctionCallExpression function) throws AlgebricksException {
+ // skip external functions that are not implemented in Java
+ IFunctionInfo fi = function.getFunctionInfo();
+ if (fi instanceof IExternalFunctionInfo
+ && !Function.FunctionLanguage.JAVA.name().equals(((IExternalFunctionInfo) fi).getLanguage())) {
+ return false;
+ }
// skip all functions that would produce records/arrays/multisets (derived types) in their open format
// this is because constant folding them will make them closed (currently)
if (function.getFunctionIdentifier().equals(BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR)) {
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java
index c0d6f82..b93335b 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java
@@ -35,6 +35,7 @@
import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
@@ -77,6 +78,11 @@
}
@Override
+ protected Function.FunctionLanguage getFunctionLanguage() {
+ return Function.FunctionLanguage.AQL;
+ }
+
+ @Override
public Pair<ILogicalOperator, LogicalVariable> visit(ForClause fc, Mutable<ILogicalOperator> tupSource)
throws CompilationException {
LogicalVariable v = context.newVarFromExpression(fc.getVarExpr());
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index 9df621c..f3480d5 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -72,7 +72,6 @@
import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.lang.common.util.RangeMapBuilder;
import org.apache.asterix.lang.common.visitor.base.AbstractQueryExpressionVisitor;
-import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.declared.DataSource;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.DatasetDataSource;
@@ -166,7 +165,7 @@
* source for the current subtree.
*/
-class LangExpressionToPlanTranslator
+abstract class LangExpressionToPlanTranslator
extends AbstractQueryExpressionVisitor<Pair<ILogicalOperator, LogicalVariable>, Mutable<ILogicalOperator>>
implements ILangExpressionToPlanTranslator {
@@ -885,22 +884,20 @@
return f;
}
+ protected abstract Function.FunctionLanguage getFunctionLanguage();
+
private AbstractFunctionCallExpression lookupUserDefinedFunction(FunctionSignature signature,
List<Mutable<ILogicalExpression>> args, SourceLocation sourceLoc) throws CompilationException {
try {
- if (signature.getDataverseName() == null) {
- return null;
- }
Function function =
- MetadataManager.INSTANCE.getFunction(metadataProvider.getMetadataTxnContext(), signature);
+ FunctionUtil.lookupUserDefinedFunctionDecl(metadataProvider.getMetadataTxnContext(), signature);
if (function == null) {
return null;
}
IFunctionInfo finfo =
- function.getLanguage().isExternal()
- ? ExternalFunctionCompilerUtil
- .getExternalFunctionInfo(metadataProvider.getMetadataTxnContext(), function)
- : FunctionUtil.getFunctionInfo(signature);
+ getFunctionLanguage().equals(function.getLanguage()) ? FunctionUtil.getFunctionInfo(signature)
+ : ExternalFunctionCompilerUtil
+ .getExternalFunctionInfo(metadataProvider.getMetadataTxnContext(), function);
AbstractFunctionCallExpression f = new ScalarFunctionCallExpression(finfo, args);
f.setSourceLocation(sourceLoc);
return f;
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
index 82dc344..3023305 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
@@ -82,6 +82,7 @@
import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
import org.apache.asterix.lang.sqlpp.visitor.base.ISqlppVisitor;
import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.om.base.ABoolean;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.base.AString;
@@ -161,6 +162,11 @@
}
@Override
+ protected Function.FunctionLanguage getFunctionLanguage() {
+ return Function.FunctionLanguage.SQLPP;
+ }
+
+ @Override
public Pair<ILogicalOperator, LogicalVariable> visit(Query q, Mutable<ILogicalOperator> tupSource)
throws CompilationException {
Expression queryBody = q.getBody();
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 93fcc7e..dcf2f2d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -23,7 +23,6 @@
import java.io.InputStream;
import java.rmi.RemoteException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
@@ -33,12 +32,12 @@
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ExecutorService;
-import java.util.stream.Collectors;
import org.apache.asterix.active.ActivityState;
import org.apache.asterix.active.EntityId;
@@ -1840,13 +1839,20 @@
}
}
if (cfs.isExternal()) {
- Function.FunctionLanguage functionLang = Function.FunctionLanguage.findByName(cfs.getLang());
- if (functionLang == null || !functionLang.isExternal()) {
- String expectedExternalLanguages = Arrays.stream(Function.FunctionLanguage.values())
- .filter(Function.FunctionLanguage::isExternal).map(Function.FunctionLanguage::getName)
- .collect(Collectors.joining(" or "));
+ String lang = cfs.getLang();
+ if (lang == null) {
+ throw new CompilationException(ErrorCode.COMPILATION_INCOMPATIBLE_FUNCTION_LANGUAGE, sourceLoc, "");
+ }
+ Function.FunctionLanguage functionLang;
+ try {
+ functionLang = Function.FunctionLanguage.valueOf(lang.toUpperCase(Locale.ROOT));
+ } catch (IllegalArgumentException e) {
throw new CompilationException(ErrorCode.COMPILATION_INCOMPATIBLE_FUNCTION_LANGUAGE, sourceLoc,
- expectedExternalLanguages, cfs.getLang());
+ lang);
+ }
+ if (functionLang.equals(getFunctionLanguage())) {
+ throw new CompilationException(ErrorCode.COMPILATION_INCOMPATIBLE_FUNCTION_LANGUAGE, sourceLoc,
+ lang);
}
Library libraryInMetadata = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverseName, libraryName);
if (libraryInMetadata == null) {
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.0.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.0.ddl.sqlpp
new file mode 100644
index 0000000..5a5bbec
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.0.ddl.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE externallibtest if exists;
+CREATE DATAVERSE externallibtest;
+USE externallibtest;
+
+create type CountryCapitalType if not exists as closed {
+country: string,
+capital: string
+};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.1.lib.sqlpp
new file mode 100644
index 0000000..d1e0e87
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.1.lib.sqlpp
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+install externallibtest testlib target/data/externallib/asterix-external-data-testlib.zip
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.2.ddl.sqlpp
new file mode 100644
index 0000000..8439cfb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.2.ddl.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use externallibtest;
+
+create function getCapital_default(a: string) returns CountryCapitalType
+language java as "testlib","org.apache.asterix.external.library.CapitalFinderFactory";
+
+create function getCapital_deterministic(a: string) returns CountryCapitalType
+language java deterministic as "testlib","org.apache.asterix.external.library.CapitalFinderFactory";
+
+create function getCapital_not_deterministic(a: string) returns CountryCapitalType
+language java not deterministic as "testlib","org.apache.asterix.external.library.CapitalFinderFactory";
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.3.query.sqlpp
new file mode 100644
index 0000000..4b2de03
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.3.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use externallibtest;
+
+{
+ "default": getCapital_default("United States").capital,
+ "deterministic": getCapital_deterministic("United States").capital,
+ "not_deterministic": getCapital_not_deterministic("United States").capital
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.4.query.sqlpp
new file mode 100644
index 0000000..be157fd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.4.query.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/** Test that external functions declared as deterministic are constant folded,
+ while non-deterministic ones are not */
+
+use externallibtest;
+
+explain
+{
+ "default": getCapital_default("United States"),
+ "deterministic": getCapital_deterministic("United States"),
+ "not_deterministic": getCapital_not_deterministic("United States")
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.lib.sqlpp
new file mode 100644
index 0000000..86af80f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.lib.sqlpp
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+uninstall externallibtest testlib
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.6.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.6.ddl.sqlpp
new file mode 100644
index 0000000..cb57494
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.6.ddl.sqlpp
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE externallibtest;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/deterministic/deterministic.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/deterministic/deterministic.3.adm
new file mode 100644
index 0000000..8eb8a8a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/deterministic/deterministic.3.adm
@@ -0,0 +1 @@
+{ "default": "Washington D.C.", "deterministic": "Washington D.C.", "not_deterministic": "Washington D.C." }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/deterministic/deterministic.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/deterministic/deterministic.4.adm
new file mode 100644
index 0000000..18dce53
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/deterministic/deterministic.4.adm
@@ -0,0 +1,8 @@
+distribute result [$$1]
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ assign [$$1] <- [{"default": getCapital_default("United States"), "deterministic": { country: "United States", capital: "Washington D.C." }, "not_deterministic": getCapital_not_deterministic("United States")}]
+ -- ASSIGN |UNPARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
index 532b8e8..31b1c13 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
@@ -59,6 +59,11 @@
<output-dir compare="Text">upperCase</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-library">
+ <compilation-unit name="deterministic">
+ <output-dir compare="Text">deterministic</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="feeds">
<compilation-unit name="feed-with-external-function">
<output-dir compare="Text">feed-with-external-function</output-dir>
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 257be8c..5dcc8ec 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -138,7 +138,7 @@
1052 = Cannot create index with the same field \"%1$s\" specified more than once.
1053 = Cannot create primary index on external dataset.
1054 = Compilation failed due to some problem in the query plan.
-1055 = Incompatible function language. Expect %1$s, but %2$s found.
+1055 = Incompatible function language: %1$s.
1056 = Too many options were specified for %1$s
1057 = Expression of type %1$s is not supported in constant record
1058 = Literal of type %1$s is not supported in constant record
diff --git a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
index a6fde0c..d94500d 100644
--- a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
+++ b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
@@ -129,7 +129,7 @@
List<FunctionDecl> storedFunctionDecls = new ArrayList<>();
for (Expression topLevelExpr : topStatement.getDirectlyEnclosedExpressions()) {
storedFunctionDecls.addAll(FunctionUtil.retrieveUsedStoredFunctions(metadataProvider, topLevelExpr, funIds,
- null, expr -> getFunctionCalls(expr), func -> functionParser.getFunctionDecl(func),
+ null, expr -> getFunctionCalls(expr), functionParser,
(signature, sourceLoc) -> CommonFunctionMapUtil.normalizeBuiltinFunctionSignature(signature)));
declaredFunctions.addAll(storedFunctionDecls);
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/FunctionParser.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/FunctionParser.java
index 2a717a9..3a5d488 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/FunctionParser.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/FunctionParser.java
@@ -39,10 +39,14 @@
this.parserFactory = parserFactory;
}
+ public Function.FunctionLanguage getFunctionLanguage() {
+ return language;
+ }
+
public FunctionDecl getFunctionDecl(Function function) throws CompilationException {
if (!function.getLanguage().equals(language)) {
- throw new CompilationException(ErrorCode.COMPILATION_INCOMPATIBLE_FUNCTION_LANGUAGE, language.getName(),
- function.getLanguage().getName());
+ throw new CompilationException(ErrorCode.COMPILATION_INCOMPATIBLE_FUNCTION_LANGUAGE, language,
+ function.getLanguage());
}
IParser parser = parserFactory.createParser(new StringReader(function.getFunctionBody()));
return parser.parseFunctionBody(function.getSignature(), function.getArgNames());
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
index 5308b96..c9c8abc 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
@@ -40,6 +40,7 @@
import org.apache.asterix.lang.common.expression.TypeExpression;
import org.apache.asterix.lang.common.expression.TypeReferenceExpression;
import org.apache.asterix.lang.common.expression.UnorderedListTypeDefinition;
+import org.apache.asterix.lang.common.parser.FunctionParser;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
@@ -118,11 +119,6 @@
}
@FunctionalInterface
- public interface IFunctionParser {
- FunctionDecl getFunctionDecl(Function function) throws CompilationException;
- }
-
- @FunctionalInterface
public interface IFunctionNormalizer {
FunctionSignature normalizeBuiltinFunctionSignature(FunctionSignature fs, SourceLocation sourceLoc)
throws CompilationException;
@@ -149,8 +145,8 @@
*/
public static List<FunctionDecl> retrieveUsedStoredFunctions(MetadataProvider metadataProvider,
Expression expression, List<FunctionSignature> declaredFunctions, List<FunctionDecl> inputFunctionDecls,
- IFunctionCollector functionCollector, IFunctionParser functionParser,
- IFunctionNormalizer functionNormalizer) throws CompilationException {
+ IFunctionCollector functionCollector, FunctionParser functionParser, IFunctionNormalizer functionNormalizer)
+ throws CompilationException {
List<FunctionDecl> functionDecls =
inputFunctionDecls == null ? new ArrayList<>() : new ArrayList<>(inputFunctionDecls);
if (expression == null) {
@@ -204,7 +200,7 @@
messageBuilder.toString());
}
- if (!function.getLanguage().isExternal()) {
+ if (functionParser.getFunctionLanguage().equals(function.getLanguage())) {
FunctionDecl functionDecl = functionParser.getFunctionDecl(function);
if (functionDecl != null) {
if (functionDecls.contains(functionDecl)) {
@@ -266,7 +262,7 @@
return dependencies;
}
- private static Function lookupUserDefinedFunctionDecl(MetadataTransactionContext mdTxnCtx,
+ public static Function lookupUserDefinedFunctionDecl(MetadataTransactionContext mdTxnCtx,
FunctionSignature signature) throws AlgebricksException {
if (signature.getDataverseName() == null) {
return null;
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
index b439ba1..dc39425 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
@@ -91,7 +91,7 @@
public static final String INLINE_WITH_OPTION = "inline_with";
private static final boolean INLINE_WITH_OPTION_DEFAULT = true;
private final IParserFactory parserFactory;
- private final FunctionParser functionRepository;
+ private final FunctionParser functionParser;
private IReturningStatement topExpr;
private List<FunctionDecl> declaredFunctions;
private LangRewritingContext context;
@@ -101,7 +101,7 @@
public SqlppQueryRewriter(IParserFactory parserFactory) {
this.parserFactory = parserFactory;
- functionRepository = new FunctionParser(Function.FunctionLanguage.SQLPP, parserFactory);
+ functionParser = new FunctionParser(Function.FunctionLanguage.SQLPP, parserFactory);
}
protected void setup(List<FunctionDecl> declaredFunctions, IReturningStatement topExpr,
@@ -196,7 +196,8 @@
return;
}
// Inlines with expressions.
- InlineWithExpressionVisitor inlineWithExpressionVisitor = new InlineWithExpressionVisitor(context);
+ InlineWithExpressionVisitor inlineWithExpressionVisitor =
+ new InlineWithExpressionVisitor(context, metadataProvider);
rewriteTopExpr(inlineWithExpressionVisitor, null);
}
@@ -261,10 +262,10 @@
List<FunctionDecl> usedStoredFunctionDecls = new ArrayList<>();
for (Expression topLevelExpr : topExpr.getDirectlyEnclosedExpressions()) {
- usedStoredFunctionDecls.addAll(FunctionUtil.retrieveUsedStoredFunctions(metadataProvider, topLevelExpr,
- funIds, null, expr -> getFunctionCalls(expr), func -> functionRepository.getFunctionDecl(func),
- (signature, sourceLoc) -> FunctionMapUtil.normalizeBuiltinFunctionSignature(signature, false,
- sourceLoc)));
+ usedStoredFunctionDecls
+ .addAll(FunctionUtil.retrieveUsedStoredFunctions(metadataProvider, topLevelExpr, funIds, null,
+ expr -> getFunctionCalls(expr), functionParser, (signature, sourceLoc) -> FunctionMapUtil
+ .normalizeBuiltinFunctionSignature(signature, false, sourceLoc)));
}
declaredFunctions.addAll(usedStoredFunctionDecls);
if (inlineUdfs && !declaredFunctions.isEmpty()) {
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/InlineWithExpressionVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/InlineWithExpressionVisitor.java
index 9865bdf..b46b9f9 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/InlineWithExpressionVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/InlineWithExpressionVisitor.java
@@ -34,14 +34,15 @@
import org.apache.asterix.lang.sqlpp.util.SqlppRewriteUtil;
import org.apache.asterix.lang.sqlpp.visitor.CheckNonFunctionalExpressionVisitor;
import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppExpressionScopingVisitor;
+import org.apache.asterix.metadata.declared.MetadataProvider;
public class InlineWithExpressionVisitor extends AbstractSqlppExpressionScopingVisitor {
- private final CheckNonFunctionalExpressionVisitor checkNonFunctionalExpressionVisitor =
- new CheckNonFunctionalExpressionVisitor();
+ private final CheckNonFunctionalExpressionVisitor checkNonFunctionalExpressionVisitor;
- public InlineWithExpressionVisitor(LangRewritingContext context) {
+ public InlineWithExpressionVisitor(LangRewritingContext context, MetadataProvider metadataProvider) {
super(context);
+ checkNonFunctionalExpressionVisitor = new CheckNonFunctionalExpressionVisitor(metadataProvider);
}
@Override
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckNonFunctionalExpressionVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckNonFunctionalExpressionVisitor.java
index 167660a..7a8f47f 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckNonFunctionalExpressionVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckNonFunctionalExpressionVisitor.java
@@ -20,24 +20,45 @@
package org.apache.asterix.lang.sqlpp.visitor;
import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.common.expression.CallExpr;
import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppContainsExpressionVisitor;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Function;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
/**
* Checks whether given expression is non-functional (i.e. whether it calls a non-functional function)
*/
public final class CheckNonFunctionalExpressionVisitor extends AbstractSqlppContainsExpressionVisitor<Void> {
+
+ private final MetadataProvider metadataProvider;
+
+ public CheckNonFunctionalExpressionVisitor(MetadataProvider metadataProvider) {
+ this.metadataProvider = metadataProvider;
+ }
+
@Override
public Boolean visit(CallExpr callExpr, Void arg) throws CompilationException {
FunctionSignature fs = callExpr.getFunctionSignature();
IFunctionInfo fi = FunctionUtil.getBuiltinFunctionInfo(fs.getName(), fs.getArity());
- // TODO: all external functions are considered functional for now.
- // we'll need to revisit this code once we enable non-functional in ExternalFunctionInfo
- if (fi != null && !fi.isFunctional()) {
- return true;
+ if (fi != null) {
+ if (!fi.isFunctional()) {
+ return true;
+ }
+ } else {
+ try {
+ Function function =
+ FunctionUtil.lookupUserDefinedFunctionDecl(metadataProvider.getMetadataTxnContext(), fs);
+ if (function != null && function.getDeterministic() != null && !function.getDeterministic()) {
+ return true;
+ }
+ } catch (AlgebricksException e) {
+ throw new CompilationException(ErrorCode.METADATA_ERROR, e, callExpr.getSourceLocation());
+ }
}
return super.visit(callExpr, arg);
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
index 815cd22..f9328b8 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
@@ -22,7 +22,6 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
import org.apache.asterix.common.functions.FunctionSignature;
@@ -138,30 +137,11 @@
return cache.dropFunction(this);
}
+ // WARNING: These values are stored in function metadata. Do not rename.
public enum FunctionLanguage {
- // WARNING: do not change these language names because
- // these values are stored in function metadata
- AQL(false),
- SQLPP(false),
- JAVA(true),
- PYTHON(true);
-
- private final boolean isExternal;
-
- FunctionLanguage(boolean isExternal) {
- this.isExternal = isExternal;
- }
-
- public boolean isExternal() {
- return isExternal;
- }
-
- public String getName() {
- return name();
- }
-
- public static FunctionLanguage findByName(String name) {
- return FunctionLanguage.valueOf(name.toUpperCase(Locale.ROOT));
- }
+ AQL,
+ SQLPP,
+ JAVA,
+ PYTHON
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
index 671fb46..65aa94c 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
@@ -41,6 +41,8 @@
import org.apache.asterix.builders.IARecordBuilder;
import org.apache.asterix.builders.OrderedListBuilder;
import org.apache.asterix.builders.RecordBuilder;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.transactions.TxnId;
@@ -126,9 +128,12 @@
.getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_DEFINITION_FIELD_INDEX)).getStringValue();
String languageValue = ((AString) functionRecord
.getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_LANGUAGE_FIELD_INDEX)).getStringValue();
- Function.FunctionLanguage language = Function.FunctionLanguage.findByName(languageValue);
- if (language == null) {
- throw new IllegalStateException(languageValue);
+
+ Function.FunctionLanguage language;
+ try {
+ language = Function.FunctionLanguage.valueOf(languageValue);
+ } catch (IllegalArgumentException e) {
+ throw new AsterixException(ErrorCode.METADATA_ERROR);
}
String functionKind =
((AString) functionRecord.getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_KIND_FIELD_INDEX))
@@ -311,7 +316,7 @@
// write field 6
fieldValue.reset();
- aString.setValue(function.getLanguage().getName());
+ aString.setValue(function.getLanguage().name());
stringSerde.serialize(aString, fieldValue.getDataOutput());
recordBuilder.addField(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_LANGUAGE_FIELD_INDEX, fieldValue);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
index 2469ff2..12c1c78 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.metadata.functions;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
@@ -51,12 +53,16 @@
private static IFunctionInfo getScalarFunctionInfo(MetadataTransactionContext txnCtx, Function function)
throws AlgebricksException {
+ if (function.getDeterministic() == null) {
+ throw new AsterixException(ErrorCode.METADATA_ERROR);
+ }
+
IAType returnType = function.getReturnType();
IResultTypeComputer typeComputer = new ExternalTypeComputer(returnType, function.getArgTypes());
return new ExternalScalarFunctionInfo(function.getSignature().createFunctionIdentifier(), returnType,
- function.getFunctionBody(), function.getLanguage().getName(), function.getLibrary(),
- function.getArgTypes(), function.getParams(), typeComputer);
+ function.getFunctionBody(), function.getLanguage().name(), function.getLibrary(),
+ function.getArgTypes(), function.getParams(), function.getDeterministic(), typeComputer);
}
private static IFunctionInfo getUnnestFunctionInfo(MetadataTransactionContext txnCtx, Function function) {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
index cdb9be5..a81bcf6 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
@@ -32,14 +32,15 @@
private static final long serialVersionUID = 1L;
public ExternalScalarFunctionInfo(String namespace, String library, String name, int arity, IAType returnType,
- String body, String language, List<IAType> argumentTypes, Map<String, String> params,
+ String body, String language, List<IAType> argumentTypes, Map<String, String> params, boolean deterministic,
IResultTypeComputer rtc) {
super(namespace, name, arity, FunctionKind.SCALAR, argumentTypes, returnType, rtc, body, language, library,
- params);
+ params, deterministic);
}
public ExternalScalarFunctionInfo(FunctionIdentifier fid, IAType returnType, String body, String language,
- String library, List<IAType> argumentTypes, Map<String, String> params, IResultTypeComputer rtc) {
- super(fid, FunctionKind.SCALAR, argumentTypes, returnType, rtc, body, language, library, params);
+ String library, List<IAType> argumentTypes, Map<String, String> params, boolean deterministic,
+ IResultTypeComputer rtc) {
+ super(fid, FunctionKind.SCALAR, argumentTypes, returnType, rtc, body, language, library, params, deterministic);
}
}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
index e76b07d..7220d05 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
@@ -41,16 +41,15 @@
public ExternalFunctionInfo(String namespace, String name, int arity, FunctionKind kind, List<IAType> argumentTypes,
IAType returnType, IResultTypeComputer rtc, String body, String language, String library,
- Map<String, String> params) {
- this(new FunctionIdentifier(namespace, name, arity), kind, argumentTypes, returnType, rtc, body, library,
- language, params);
+ Map<String, String> params, boolean deterministic) {
+ this(new FunctionIdentifier(namespace, name, arity), kind, argumentTypes, returnType, rtc, body, language,
+ library, params, deterministic);
}
public ExternalFunctionInfo(FunctionIdentifier fid, FunctionKind kind, List<IAType> argumentTypes,
IAType returnType, IResultTypeComputer rtc, String body, String language, String library,
- Map<String, String> params) {
- // TODO: fix CheckNonFunctionalExpressionVisitor once we have non-functional external functions
- super(fid, true);
+ Map<String, String> params, boolean deterministic) {
+ super(fid, deterministic);
this.rtc = rtc;
this.argumentTypes = argumentTypes;
this.body = body;