Add install/uninstall UDF to Ansible
1. Add udf.sh for handling install/uninstall UDF packages to the cluster.
2. Remove unnecessary path check in TestLibrarian.
Change-Id: If7ea9640a06e2b691c19a8a819307d84b55a679e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1682
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
index b4e1c75..5f86c28 100755
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
@@ -50,11 +50,11 @@
import org.apache.asterix.metadata.entities.Library;
import org.apache.asterix.metadata.utils.MetadataUtil;
import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
-import org.apache.hyracks.control.common.controllers.ControllerConfig;
public class ExternalLibraryUtils {
private static final Logger LOGGER = Logger.getLogger(ExternalLibraryUtils.class.getName());
+ private static final FilenameFilter nonHiddenFileNameFilter = (dir, name) -> !name.startsWith(".");
private ExternalLibraryUtils() {
}
@@ -72,19 +72,15 @@
// directory exists?
if (installLibDir.exists()) {
// get the list of files in the directory
- for (String dataverse : installLibDir.list()) {
- File dataverseDir = new File(installLibDir, dataverse);
- String[] libraries = dataverseDir.list();
- for (String library : libraries) {
+ for (File dataverseDir : installLibDir.listFiles(File::isDirectory)) {
+ for (File libraryDir : dataverseDir.listFiles(File::isDirectory)) {
// for each file (library), register library
- registerLibrary(externalLibraryManager, dataverse, library);
+ registerLibrary(externalLibraryManager, dataverseDir.getName(), libraryDir.getName());
// is metadata node?
if (isMetadataNode) {
// get library file
- File libraryDir = new File(installLibDir.getAbsolutePath() + File.separator + dataverse
- + File.separator + library);
// install if needed (i,e, add the functions, adapters, datasources, parsers to the metadata)
- installLibraryIfNeeded(dataverse, libraryDir, uninstalledLibs);
+ installLibraryIfNeeded(dataverseDir.getName(), libraryDir, uninstalledLibs);
}
}
}
@@ -105,7 +101,7 @@
// directory exists?
if (uninstallLibDir.exists()) {
// list files
- uninstallLibNames = uninstallLibDir.list();
+ uninstallLibNames = uninstallLibDir.list(nonHiddenFileNameFilter);
for (String uninstallLibName : uninstallLibNames) {
// Get the <dataverse name - library name> pair
String[] components = uninstallLibName.split("\\.");
@@ -295,7 +291,6 @@
*
* @param dataverse
* @param libraryName
- * @param installLibDir
* @throws Exception
*/
protected static void registerLibrary(ILibraryManager externalLibraryManager, String dataverse, String libraryName)
@@ -395,16 +390,26 @@
* @return the directory "$(ControllerConfig.defaultDir)/library": This needs to be improved
*/
protected static File getLibraryInstallDir() {
- String workingDir = System.getProperty("user.dir");
- return new File(workingDir, "library");
+ // Check managix directory first. If not exists, check app home.
+ File installDir = new File(System.getProperty("user.dir"), "library");
+ if (!installDir.exists()) {
+ installDir = new File(System.getProperty("app.home", System.getProperty("user.home"))
+ + File.separator + "lib" + File.separator + "udfs");
+ }
+ return installDir;
}
/**
* @return the directory "$(ControllerConfig.defaultDir)/uninstall": This needs to be improved
*/
protected static File getLibraryUninstallDir() {
- String workingDir = System.getProperty("user.dir");
- return new File(workingDir, "uninstall");
+ // Check managix directory first. If not exists, check app home.
+ File uninstallDir = new File(System.getProperty("user.dir"), "uninstall");
+ if (!uninstallDir.exists()) {
+ uninstallDir = new File(System.getProperty("app.home", System.getProperty("user.home"))
+ + File.separator + "lib" + File.separator + "udfs" + File.separator + "uninstall");
+ }
+ return uninstallDir;
}
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
index e92b5e8..c67d26c 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
@@ -35,8 +35,6 @@
public class TestLibrarian implements ITestLibrarian {
- public static final String LIBRARY_DIR_NAME = "library";
-
// The following list includes a library manager for the CC
// and library managers for NCs (one-per-NC).
private final List<ILibraryManager> libraryManagers;
@@ -84,9 +82,6 @@
public static void removeLibraryDir() throws IOException {
File installLibDir = ExternalLibraryUtils.getLibraryInstallDir();
- if (!installLibDir.getAbsolutePath().endsWith(LIBRARY_DIR_NAME)) {
- throw new HyracksDataException("Invalid library directory");
- }
FileUtils.deleteQuietly(installLibDir);
}
diff --git a/asterixdb/asterix-server/src/main/opt/ansible/bin/udf.sh b/asterixdb/asterix-server/src/main/opt/ansible/bin/udf.sh
new file mode 100755
index 0000000..2f8b6fe
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/opt/ansible/bin/udf.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+# Get options
+usage() { echo "./udf.sh -m [i|u] -d DATAVERSE_NAME -l LIBRARY_NAME [-p UDF_PACKAGE_PATH]" 1>&2; exit 1; }
+
+while getopts ":d:l:p:m:" o; do
+ case "${o}" in
+ m)
+ mode=${OPTARG}
+ ;;
+ d)
+ dname=${OPTARG}
+ ;;
+ l)
+ libname=${OPTARG}
+ ;;
+ p)
+ tarpath=${OPTARG}
+ ;;
+ *)
+ echo $o
+ usage
+ ;;
+ esac
+done
+shift $((OPTIND-1))
+
+if [[ -z $dname ]] || [[ -z $libname ]]; then
+ echo "Dataverse name or Library name is missing"
+fi
+
+# Gets the absolute path so that the script can work no matter where it is invoked.
+pushd `dirname $0` > /dev/null
+SCRIPT_PATH=`pwd -P`
+popd > /dev/null
+ANSB_PATH=`dirname "${SCRIPT_PATH}"`
+
+INVENTORY=$ANSB_PATH/conf/inventory
+
+# Deploy/Destroy the UDF package on all nodes.
+export ANSIBLE_HOST_KEY_CHECKING=false
+if [[ $mode = "i" ]]; then
+ if [[ -z $tarpath ]]; then
+ echo "UDF package path is undefined"
+ else
+ echo "Install library to $dname.$libname"
+ ansible-playbook -i $INVENTORY $ANSB_PATH/yaml/deploy_udf.yml --extra-vars "dataverse=$dname libname=$libname package_path=$tarpath"
+ fi
+elif [[ $mode == "u" ]]; then
+ echo "Uninstall library in $dname.$libname"
+ ansible-playbook -i $INVENTORY $ANSB_PATH/yaml/destroy_udf.yml --extra-vars "dataverse=$dname libname=$libname"
+else
+ echo "Wrong mode"
+fi
\ No newline at end of file
diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy_udf.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy_udf.yml
new file mode 100644
index 0000000..e9300ea
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy_udf.yml
@@ -0,0 +1,30 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- name: Deploy the UDF library {{ dataverse }}.{{ libname }} to the cluster
+ hosts: all
+ tasks:
+ - include_vars: ../conf/instance_settings.yml
+ - file:
+ path: "{{ binarydir }}/lib/udfs/{{ dataverse }}/{{ libname }}"
+ state: directory
+ mode: 0755
+ - unarchive:
+ src: "{{ package_path }}"
+ dest: "{{ binarydir }}/lib/udfs/{{ dataverse }}/{{ libname }}"
diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/destroy_udf.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/destroy_udf.yml
new file mode 100644
index 0000000..cc4add3
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/destroy_udf.yml
@@ -0,0 +1,33 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- name: Destroy the library {{ libname }} in dataverse {{ dataverse }}
+ hosts: all
+ tasks:
+ - include_vars: ../conf/instance_settings.yml
+ - file:
+ path: "{{ binarydir }}/lib/udfs/uninstall/"
+ state: directory
+ mode: 0755
+ - file:
+ state: touch
+ path: "{{ binarydir }}/lib/udfs/uninstall/{{ dataverse }}.{{ libname }}"
+ - file:
+ path: "{{ binarydir }}/lib/udfs/{{ dataverse }}/{{ libname }}"
+ state: absent