Fix function usage check in feed
1. Add reference count field to Function entity.
2. Add reference count check when drop function.
3. Increate reference count when apply function to feed. Decrease the
reference count when disconnect feed.
4. Add test cases for drop function that is being used and drop function
that is no longer in use.
Change-Id: I04bed3836455997269a4cae7088dcc25c745a3df
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1763
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
index 5f86c28..0e72976 100755
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
@@ -247,7 +247,7 @@
}
Function f = new Function(dataverse, libraryName + "#" + function.getName().trim(), args.size(),
args, function.getReturnType().trim(), function.getDefinition().trim(),
- library.getLanguage().trim(), function.getFunctionType().trim());
+ library.getLanguage().trim(), function.getFunctionType().trim(), 0);
MetadataManager.INSTANCE.addFunction(mdTxnCtx, f);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Installed function: " + libraryName + "#" + function.getName().trim());
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index e9c8cf7..f64812e 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -1642,9 +1642,10 @@
if (dv == null) {
throw new AlgebricksException("There is no dataverse with this name " + dataverse + ".");
}
+ // If the function body contains function calls, theirs reference count won't be increased.
Function function = new Function(dataverse, functionName, cfs.getaAterixFunction().getArity(),
cfs.getParamList(), Function.RETURNTYPE_VOID, cfs.getFunctionBody(), Function.LANGUAGE_AQL,
- FunctionKind.SCALAR.toString());
+ FunctionKind.SCALAR.toString(), 0);
MetadataManager.INSTANCE.addFunction(mdTxnCtx, function);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -1670,6 +1671,8 @@
if (!stmtDropFunction.getIfExists()) {
throw new AlgebricksException("Unknonw function " + signature);
}
+ } else if (function.getReferenceCount() != 0) {
+ throw new AlgebricksException("Function " + signature + " is being used. It cannot be dropped.");
} else {
MetadataManager.INSTANCE.dropFunction(mdTxnCtx, signature);
}
@@ -2153,6 +2156,13 @@
fc = new FeedConnection(dataverseName, feedName, datasetName, appliedFunctions, policyName,
outputType.toString());
MetadataManager.INSTANCE.addFeedConnection(metadataProvider.getMetadataTxnContext(), fc);
+ // Increase function reference count.
+ for (FunctionSignature funcSig : appliedFunctions) {
+ // The function should be cached in Metadata manager, so this operation is not that expensive.
+ Function func = MetadataManager.INSTANCE.getFunction(mdTxnCtx, funcSig);
+ func.reference();
+ MetadataManager.INSTANCE.updateFunction(mdTxnCtx, func);
+ }
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
} catch (Exception e) {
abort(e, e, mdTxnCtx);
@@ -2189,6 +2199,12 @@
+ cfs.getDatasetName().getValue() + ". Invalid operation!");
}
MetadataManager.INSTANCE.dropFeedConnection(mdTxnCtx, dataverseName, feedName, datasetName);
+ for (FunctionSignature functionSignature : fc.getAppliedFunctions()) {
+ Function function = MetadataManager.INSTANCE.getFunction(mdTxnCtx, functionSignature);
+ function.dereference();
+ MetadataManager.INSTANCE.updateFunction(mdTxnCtx, function);
+ }
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
} catch (Exception e) {
abort(e, e, mdTxnCtx);
throw e;
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta06/meta06.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta06/meta06.1.adm
index 58f5b77..2131611 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta06/meta06.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta06/meta06.1.adm
@@ -1 +1 @@
-{ "DataverseName": "testdv", "Name": "fun01", "Arity": "0", "Params": [ ], "ReturnType": "VOID", "Definition": "\"This is an AQL Bodied UDF\"", "Language": "AQL", "Kind": "SCALAR" }
+{ "DataverseName": "testdv", "Name": "fun01", "Arity": "0", "Params": [ ], "ReturnType": "VOID", "Definition": "\"This is an AQL Bodied UDF\"", "Language": "AQL", "Kind": "SCALAR", "ReferenceCount": "0" }
diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm
index 7229aa8..650c924 100644
--- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm
@@ -29,7 +29,7 @@
{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "FeedName", "FieldType": "string", "IsNullable": false }, { "FieldName": "AdapterName", "FieldType": "string", "IsNullable": false }, { "FieldName": "AdapterConfiguration", "FieldType": "FeedRecordType_AdapterConfiguration", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" }
{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType_AdapterConfiguration", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "UnorderedList": "FeedRecordType_AdapterConfiguration_Item" }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" }
{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType_AdapterConfiguration_Item", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Value", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" }
-{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Arity", "FieldType": "string", "IsNullable": false }, { "FieldName": "Params", "FieldType": "FunctionRecordType_Params", "IsNullable": false }, { "FieldName": "ReturnType", "FieldType": "string", "IsNullable": false }, { "FieldName": "Definition", "FieldType": "string", "IsNullable": false }, { "FieldName": "Language", "FieldType": "string", "IsNullable": false }, { "FieldName": "Kind", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" }
+{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Arity", "FieldType": "string", "IsNullable": false }, { "FieldName": "Params", "FieldType": "FunctionRecordType_Params", "IsNullable": false }, { "FieldName": "ReturnType", "FieldType": "string", "IsNullable": false }, { "FieldName": "Definition", "FieldType": "string", "IsNullable": false }, { "FieldName": "Language", "FieldType": "string", "IsNullable": false }, { "FieldName": "Kind", "FieldType": "string", "IsNullable": false }, { "FieldName": "ReferenceCount", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Sun May 21 09:56:53 PDT 2017" }
{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType_Params", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "string" }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" }
{ "DataverseName": "Metadata", "DatatypeName": "IndexRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatasetName", "FieldType": "string", "IsNullable": false }, { "FieldName": "IndexName", "FieldType": "string", "IsNullable": false }, { "FieldName": "IndexStructure", "FieldType": "string", "IsNullable": false }, { "FieldName": "SearchKey", "FieldType": "IndexRecordType_SearchKey", "IsNullable": false }, { "FieldName": "IsPrimary", "FieldType": "boolean", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false }, { "FieldName": "PendingOp", "FieldType": "int32", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" }
{ "DataverseName": "Metadata", "DatatypeName": "IndexRecordType_SearchKey", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "IndexRecordType_SearchKey_Item" }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-no-longer-used-by-feed/drop-function-no-longer-used-by-feed.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-no-longer-used-by-feed/drop-function-no-longer-used-by-feed.1.ddl.aql
new file mode 100644
index 0000000..19eda0e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-no-longer-used-by-feed/drop-function-no-longer-used-by-feed.1.ddl.aql
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse experiments if exists;
+create dataverse experiments;
+use dataverse experiments;
+
+create type TwitterUser if not exists as open{
+ screen-name: string,
+ friends_count: int32,
+ name: string,
+ followers_count: int32
+};
+
+create dataset TwitterUsers(TwitterUser) primary key screen-name;
+
+create function test_func0($xyz) {
+ let $tty1 := if ($xyz.followers_count > 25000) then {"popularity":"Good!"} else {"popularity":"Bad!"}
+ return object_merge($tty1, $xyz)
+}
+
+create feed UserFeed using socket_adapter
+(
+ ("sockets"="127.0.0.1:10001"),
+ ("address-type"="IP"),
+ ("type-name"="TwitterUser"),
+ ("format"="adm"),
+ ("upsert-feed"="true")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-no-longer-used-by-feed/drop-function-no-longer-used-by-feed.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-no-longer-used-by-feed/drop-function-no-longer-used-by-feed.2.update.aql
new file mode 100644
index 0000000..f6c40de
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-no-longer-used-by-feed/drop-function-no-longer-used-by-feed.2.update.aql
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse experiments;
+set wait-for-completion-feed "false";
+
+connect feed UserFeed to dataset TwitterUsers apply function test_func0;
+
+disconnect feed UserFeed from dataset TwitterUsers;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-no-longer-used-by-feed/drop-function-no-longer-used-by-feed.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-no-longer-used-by-feed/drop-function-no-longer-used-by-feed.3.ddl.aql
new file mode 100644
index 0000000..624c628
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-no-longer-used-by-feed/drop-function-no-longer-used-by-feed.3.ddl.aql
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse experiments;
+drop function test_func0@1;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-used-by-feed/drop-function-used-by-feed.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-used-by-feed/drop-function-used-by-feed.1.ddl.aql
new file mode 100644
index 0000000..19eda0e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-used-by-feed/drop-function-used-by-feed.1.ddl.aql
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse experiments if exists;
+create dataverse experiments;
+use dataverse experiments;
+
+create type TwitterUser if not exists as open{
+ screen-name: string,
+ friends_count: int32,
+ name: string,
+ followers_count: int32
+};
+
+create dataset TwitterUsers(TwitterUser) primary key screen-name;
+
+create function test_func0($xyz) {
+ let $tty1 := if ($xyz.followers_count > 25000) then {"popularity":"Good!"} else {"popularity":"Bad!"}
+ return object_merge($tty1, $xyz)
+}
+
+create feed UserFeed using socket_adapter
+(
+ ("sockets"="127.0.0.1:10001"),
+ ("address-type"="IP"),
+ ("type-name"="TwitterUser"),
+ ("format"="adm"),
+ ("upsert-feed"="true")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-used-by-feed/drop-function-used-by-feed.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-used-by-feed/drop-function-used-by-feed.2.update.aql
new file mode 100644
index 0000000..4654612
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-used-by-feed/drop-function-used-by-feed.2.update.aql
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse experiments;
+set wait-for-completion-feed "false";
+
+connect feed UserFeed to dataset TwitterUsers apply function test_func0;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-used-by-feed/drop-function-used-by-feed.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-used-by-feed/drop-function-used-by-feed.3.ddl.aql
new file mode 100644
index 0000000..624c628
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-function-used-by-feed/drop-function-used-by-feed.3.ddl.aql
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse experiments;
+drop function test_func0@1;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/drop-function-used-by-feed/drop-function-used-by-feed.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/drop-function-used-by-feed/drop-function-used-by-feed.1.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/drop-function-used-by-feed/drop-function-used-by-feed.1.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/single-line-definition/single-line-definition.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/single-line-definition/single-line-definition.1.adm
index 5ba00ae..5545dfb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/single-line-definition/single-line-definition.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/single-line-definition/single-line-definition.1.adm
@@ -1 +1 @@
-{ "DataverseName": "test", "Name": "printName", "Arity": "0", "Params": [ ], "ReturnType": "VOID", "Definition": "'AsterixDB Shared nothing parallel BDMS'", "Language": "AQL", "Kind": "SCALAR" }
+{ "DataverseName": "test", "Name": "printName", "Arity": "0", "Params": [ ], "ReturnType": "VOID", "Definition": "'AsterixDB Shared nothing parallel BDMS'", "Language": "AQL", "Kind": "SCALAR", "ReferenceCount": "0" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf28/udf28.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf28/udf28.1.adm
index b106dbe..2bc8eaf 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf28/udf28.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf28/udf28.1.adm
@@ -1 +1 @@
-{ "DataverseName": "test", "Name": "f1", "Arity": "0", "Params": [ ], "ReturnType": "VOID", "Definition": "100", "Language": "AQL", "Kind": "SCALAR" }
+{ "DataverseName": "test", "Name": "f1", "Arity": "0", "Params": [ ], "ReturnType": "VOID", "Definition": "100", "Language": "AQL", "Kind": "SCALAR", "ReferenceCount": "0" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
index e2070c0..d5b8e74 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -279,6 +279,17 @@
<expected-error>Cannot find function</expected-error>
</compilation-unit>
</test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="drop-function-used-by-feed">
+ <output-dir compare="Text">drop-function-used-by-feed</output-dir>
+ <expected-error>Function experiments.test_func0@1 is being used. It cannot be dropped.</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
+ <compilation-unit name="drop-function-no-longer-used-by-feed">
+ <output-dir compare="Text">drop-function-used-by-feed</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="upsert">
<test-case FilePath="upsert">
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/functionDataset/functionDataset.1.adm b/asterixdb/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/functionDataset/functionDataset.1.adm
index ef563f1..52fb855 100644
--- a/asterixdb/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/functionDataset/functionDataset.1.adm
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/functionDataset/functionDataset.1.adm
@@ -1,8 +1,8 @@
-{ "DataverseName": "externallibtest", "Name": "testlib#addHashTags", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsFactory", "Language": "JAVA", "Kind": "SCALAR" }
-{ "DataverseName": "externallibtest", "Name": "testlib#addHashTagsInPlace", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsInPlaceFactory", "Language": "JAVA", "Kind": "SCALAR" }
-{ "DataverseName": "externallibtest", "Name": "testlib#allTypes", "Arity": "1", "Params": [ "AllType" ], "ReturnType": "AllType", "Definition": "org.apache.asterix.external.library.AllTypesFactory", "Language": "JAVA", "Kind": "SCALAR" }
-{ "DataverseName": "externallibtest", "Name": "testlib#echoDelay", "Arity": "1", "Params": [ "TweetMessageType" ], "ReturnType": "TweetMessageType", "Definition": "org.apache.asterix.external.library.EchoDelayFactory", "Language": "JAVA", "Kind": "SCALAR" }
-{ "DataverseName": "externallibtest", "Name": "testlib#getCapital", "Arity": "1", "Params": [ "ASTRING" ], "ReturnType": "CountryCapitalType", "Definition": "org.apache.asterix.external.library.CapitalFinderFactory", "Language": "JAVA", "Kind": "SCALAR" }
-{ "DataverseName": "externallibtest", "Name": "testlib#mysum", "Arity": "2", "Params": [ "AINT32", "AINT32" ], "ReturnType": "AINT32", "Definition": "org.apache.asterix.external.library.SumFactory", "Language": "JAVA", "Kind": "SCALAR" }
-{ "DataverseName": "externallibtest", "Name": "testlib#parseTweet", "Arity": "1", "Params": [ "TweetInputType" ], "ReturnType": "TweetOutputType", "Definition": "org.apache.asterix.external.library.ParseTweetFactory", "Language": "JAVA", "Kind": "SCALAR" }
-{ "DataverseName": "externallibtest", "Name": "testlib#toUpper", "Arity": "1", "Params": [ "TextType" ], "ReturnType": "TextType", "Definition": "org.apache.asterix.external.library.UpperCaseFactory", "Language": "JAVA", "Kind": "SCALAR" }
+{ "DataverseName": "externallibtest", "Name": "testlib#addHashTags", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" }
+{ "DataverseName": "externallibtest", "Name": "testlib#addHashTagsInPlace", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsInPlaceFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" }
+{ "DataverseName": "externallibtest", "Name": "testlib#allTypes", "Arity": "1", "Params": [ "AllType" ], "ReturnType": "AllType", "Definition": "org.apache.asterix.external.library.AllTypesFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" }
+{ "DataverseName": "externallibtest", "Name": "testlib#echoDelay", "Arity": "1", "Params": [ "TweetMessageType" ], "ReturnType": "TweetMessageType", "Definition": "org.apache.asterix.external.library.EchoDelayFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" }
+{ "DataverseName": "externallibtest", "Name": "testlib#getCapital", "Arity": "1", "Params": [ "ASTRING" ], "ReturnType": "CountryCapitalType", "Definition": "org.apache.asterix.external.library.CapitalFinderFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" }
+{ "DataverseName": "externallibtest", "Name": "testlib#mysum", "Arity": "2", "Params": [ "AINT32", "AINT32" ], "ReturnType": "AINT32", "Definition": "org.apache.asterix.external.library.SumFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" }
+{ "DataverseName": "externallibtest", "Name": "testlib#parseTweet", "Arity": "1", "Params": [ "TweetInputType" ], "ReturnType": "TweetOutputType", "Definition": "org.apache.asterix.external.library.ParseTweetFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" }
+{ "DataverseName": "externallibtest", "Name": "testlib#toUpper", "Arity": "1", "Params": [ "TextType" ], "ReturnType": "TextType", "Definition": "org.apache.asterix.external.library.UpperCaseFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" }
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
index b2c2e81..05bf08e 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
@@ -558,6 +558,17 @@
}
@Override
+ public void updateFunction(MetadataTransactionContext mdTxnCtx, Function function) throws MetadataException {
+ try {
+ metadataNode.updateFunction(mdTxnCtx.getJobId(), function);
+ } catch (RemoteException e) {
+ throw new MetadataException(e);
+ }
+ mdTxnCtx.dropFunction(function);
+ mdTxnCtx.addFunction(function);
+ }
+
+ @Override
public void addFunction(MetadataTransactionContext mdTxnCtx, Function function) throws MetadataException {
try {
metadataNode.addFunction(mdTxnCtx.getJobId(), function);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index c4c5098..ea7a47c 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -1866,4 +1866,24 @@
throw new MetadataException(e);
}
}
+
+ @Override
+ public void updateFunction(JobId jobId, Function function) throws MetadataException, RemoteException {
+ try {
+ // remove old function
+ ITupleReference searchKey;
+ searchKey = createTuple(function.getDataverseName(), function.getName(),
+ Integer.toString(function.getArity()));
+ ITupleReference functionTuple = getTupleToBeDeleted(jobId, MetadataPrimaryIndexes.FUNCTION_DATASET,
+ searchKey);
+ deleteTupleFromIndex(jobId, MetadataPrimaryIndexes.FUNCTION_DATASET, functionTuple);
+ // add new function
+ FunctionTupleTranslator functionTupleTranslator = tupleTranslatorProvider.getFunctionTupleTranslator(true);
+ functionTuple = functionTupleTranslator.getTupleFromMetadataEntity(function);
+ insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.FUNCTION_DATASET, functionTuple);
+ } catch (HyracksDataException | ACIDException e) {
+ throw new MetadataException(e);
+ }
+
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
index b2ec7f2..d8bf559 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
@@ -158,7 +158,7 @@
public void dropFunction(FunctionSignature signature) {
Function function = new Function(signature.getNamespace(), signature.getName(), signature.getArity(), null,
- null, null, null, null);
+ null, null, null, null, 0);
droppedCache.addFunctionIfNotExists(function);
logAndApply(new MetadataLogicalOperation(function, false));
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
index 8ce53d0..f36d510 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
@@ -397,6 +397,8 @@
*/
void dropFunction(MetadataTransactionContext ctx, FunctionSignature functionSignature) throws MetadataException;
+ void updateFunction(MetadataTransactionContext ctx, Function function) throws MetadataException;
+
/**
* @param mdTxnCtx
* MetadataTransactionContext of an active metadata transaction.
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
index 21b170d..c4ae70c 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
@@ -426,6 +426,8 @@
*/
void addFunction(JobId jobId, Function function) throws MetadataException, RemoteException;
+ void updateFunction(JobId jobId, Function function) throws MetadataException, RemoteException;
+
/**
* @param ctx
* @param dataverseName
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
index 2a04b58..852c01f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
@@ -99,6 +99,7 @@
public static final String FIELD_NAME_VALUE = "Value";
public static final String FIELD_NAME_WORKING_MEMORY_SIZE = "WorkingMemorySize";
public static final String FIELD_NAME_APPLIED_FUNCTIONS = "AppliedFunctions";
+ public static final String FIELD_NAME_REFERENCE_COUNT = "ReferenceCount";
//---------------------------------- Record Types Creation ----------------------------------//
//--------------------------------------- Properties ----------------------------------------//
@@ -321,16 +322,18 @@
public static final int FUNCTION_ARECORD_FUNCTION_DEFINITION_FIELD_INDEX = 5;
public static final int FUNCTION_ARECORD_FUNCTION_LANGUAGE_FIELD_INDEX = 6;
public static final int FUNCTION_ARECORD_FUNCTION_KIND_FIELD_INDEX = 7;
+ public static final int FUNCTION_ARECORD_FUNCTION_REFERENCE_COUNT_INDEX = 8;
public static final ARecordType FUNCTION_RECORDTYPE = createRecordType(
// RecordTypeName
RECORD_NAME_FUNCTION,
// FieldNames
new String[] { FIELD_NAME_DATAVERSE_NAME, FIELD_NAME_NAME, FIELD_NAME_ARITY, FIELD_NAME_PARAMS,
- FIELD_NAME_RETURN_TYPE, FIELD_NAME_DEFINITION, FIELD_NAME_LANGUAGE, FIELD_NAME_KIND },
+ FIELD_NAME_RETURN_TYPE, FIELD_NAME_DEFINITION, FIELD_NAME_LANGUAGE, FIELD_NAME_KIND,
+ FIELD_NAME_REFERENCE_COUNT },
// FieldTypes
new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.ASTRING,
new AOrderedListType(BuiltinType.ASTRING, null), BuiltinType.ASTRING, BuiltinType.ASTRING,
- BuiltinType.ASTRING, BuiltinType.ASTRING },
+ BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.ASTRING },
//IsOpen?
true);
//------------------------------------------ Adapter ----------------------------------------//
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
index 7ff423c..b010a8a 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
@@ -19,6 +19,7 @@
package org.apache.asterix.metadata.entities;
import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.asterix.metadata.MetadataCache;
import org.apache.asterix.metadata.api.IMetadataEntity;
@@ -39,9 +40,10 @@
private final String returnType;
private final String language;
private final String kind;
+ private AtomicInteger referenceCount;
public Function(String dataverseName, String functionName, int arity, List<String> params, String returnType,
- String functionBody, String language, String functionKind) {
+ String functionBody, String language, String functionKind, int referenceCount) {
this.dataverse = dataverseName;
this.name = functionName;
this.params = params;
@@ -50,6 +52,7 @@
this.language = language;
this.kind = functionKind;
this.arity = arity;
+ this.referenceCount = new AtomicInteger(referenceCount);
}
public String getDataverseName() {
@@ -94,4 +97,15 @@
return cache.dropFunction(this);
}
+ public int getReferenceCount() {
+ return referenceCount.get();
+ }
+
+ public int reference() {
+ return referenceCount.incrementAndGet();
+ }
+
+ public int dereference() {
+ return referenceCount.decrementAndGet();
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
index 64c4035..7cbc6ac 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
@@ -104,8 +104,10 @@
String functionKind =
((AString) functionRecord.getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_KIND_FIELD_INDEX))
.getStringValue();
+ String referenceCount = ((AString) functionRecord
+ .getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_REFERENCE_COUNT_INDEX)).getStringValue();
return new Function(dataverseName, functionName, Integer.parseInt(arity), params, returnType, definition,
- language, functionKind);
+ language, functionKind, Integer.parseInt(referenceCount));
}
@@ -185,6 +187,12 @@
stringSerde.serialize(aString, fieldValue.getDataOutput());
recordBuilder.addField(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_KIND_FIELD_INDEX, fieldValue);
+ // write field 8
+ fieldValue.reset();
+ aString.setValue(Integer.toString(function.getReferenceCount()));
+ stringSerde.serialize(aString, fieldValue.getDataOutput());
+ recordBuilder.addField(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_REFERENCE_COUNT_INDEX, fieldValue);
+
// write record
recordBuilder.write(tupleBuilder.getDataOutput(), true);
tupleBuilder.addFieldEndOffset();
diff --git a/asterixdb/asterix-yarn/src/test/resources/library/results/library-metadata/functionDataset/functionDataset.1.adm b/asterixdb/asterix-yarn/src/test/resources/library/results/library-metadata/functionDataset/functionDataset.1.adm
index 0675316..cdf7792 100644
--- a/asterixdb/asterix-yarn/src/test/resources/library/results/library-metadata/functionDataset/functionDataset.1.adm
+++ b/asterixdb/asterix-yarn/src/test/resources/library/results/library-metadata/functionDataset/functionDataset.1.adm
@@ -1,8 +1,8 @@
-{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#addHashTags", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsFactory", "Language": "JAVA", "Kind": "SCALAR" }
-{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#addHashTagsInPlace", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsInPlaceFactory", "Language": "JAVA", "Kind": "SCALAR" }
-{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#allTypes", "Arity": "1", "Params": [ "AllType" ], "ReturnType": "AllType", "Definition": "org.apache.asterix.external.library.AllTypesFactory", "Language": "JAVA", "Kind": "SCALAR" }
-{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#echoDelay", "Arity": "1", "Params": [ "TweetMessageType" ], "ReturnType": "TweetMessageType", "Definition": "org.apache.asterix.external.library.EchoDelayFactory", "Language": "JAVA", "Kind": "SCALAR" }
-{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#getCapital", "Arity": "1", "Params": [ "ASTRING" ], "ReturnType": "CountryCapitalType", "Definition": "org.apache.asterix.external.library.CapitalFinderFactory", "Language": "JAVA", "Kind": "SCALAR" }
-{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#mysum", "Arity": "2", "Params": [ "AINT32", "AINT32" ], "ReturnType": "AINT32", "Definition": "org.apache.asterix.external.library.SumFactory", "Language": "JAVA", "Kind": "SCALAR" }
-{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#parseTweet", "Arity": "1", "Params": [ "TweetInputType" ], "ReturnType": "TweetOutputType", "Definition": "org.apache.asterix.external.library.ParseTweetFactory", "Language": "JAVA", "Kind": "SCALAR" }
-{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#toUpper", "Arity": "1", "Params": [ "TextType" ], "ReturnType": "TextType", "Definition": "org.apache.asterix.external.library.UpperCaseFactory", "Language": "JAVA", "Kind": "SCALAR" }
+{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#addHashTags", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" }
+{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#addHashTagsInPlace", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsInPlaceFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" }
+{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#allTypes", "Arity": "1", "Params": [ "AllType" ], "ReturnType": "AllType", "Definition": "org.apache.asterix.external.library.AllTypesFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" }
+{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#echoDelay", "Arity": "1", "Params": [ "TweetMessageType" ], "ReturnType": "TweetMessageType", "Definition": "org.apache.asterix.external.library.EchoDelayFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" }
+{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#getCapital", "Arity": "1", "Params": [ "ASTRING" ], "ReturnType": "CountryCapitalType", "Definition": "org.apache.asterix.external.library.CapitalFinderFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" }
+{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#mysum", "Arity": "2", "Params": [ "AINT32", "AINT32" ], "ReturnType": "AINT32", "Definition": "org.apache.asterix.external.library.SumFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" }
+{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#parseTweet", "Arity": "1", "Params": [ "TweetInputType" ], "ReturnType": "TweetOutputType", "Definition": "org.apache.asterix.external.library.ParseTweetFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" }
+{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#toUpper", "Arity": "1", "Params": [ "TextType" ], "ReturnType": "TextType", "Definition": "org.apache.asterix.external.library.UpperCaseFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" }