[NO ISSUE][*DB] Enable loading external UDF for AsterixHyracksIntegrationUtil

- user model changes: no
- storage format changes: no
- interface changes: no

Add external UDF support for AsterixHyracksIntegrationUtil. Developer
may use following system properties to install external UDF for local
environment.
"external.lib" : udf package path.
"external.lib.dataverse": dataverse name (default: test).
"external.lib.libname": library name (default: testlib).

Change-Id: I333238b944b1c6d48a2b80b87a25f600522e0cc4
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2078
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
index 50c3ff6..2a8a831 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
@@ -29,11 +29,14 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import org.apache.asterix.app.external.ExternalUDFLibrarian;
 import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
+import org.apache.asterix.common.api.INcApplicationContext;
 import org.apache.asterix.common.config.GlobalConfig;
 import org.apache.asterix.common.config.PropertiesAccessor;
 import org.apache.asterix.common.dataflow.ICcApplicationContext;
 import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
 import org.apache.asterix.hyracks.bootstrap.CCApplication;
 import org.apache.asterix.hyracks.bootstrap.NCApplication;
 import org.apache.commons.io.FileUtils;
@@ -91,7 +94,7 @@
             ncApplication.registerConfig(ncConfigManager);
             nodeControllers.add(
                     new NodeControllerService(fixupIODevices(createNCConfig(nodeId, ncConfigManager)), ncApplication));
-        } ;
+        }
 
         cc.start();
 
@@ -122,6 +125,22 @@
         this.ncs = nodeControllers.toArray(new NodeControllerService[nodeControllers.size()]);
     }
 
+    public void init(boolean deleteOldInstanceData, String externalLibPath) throws Exception {
+        List<ILibraryManager> libraryManagers = new ArrayList<>();
+        ExternalUDFLibrarian librarian = new ExternalUDFLibrarian(libraryManagers);
+        librarian.cleanup();
+        init(deleteOldInstanceData);
+        if (externalLibPath != null && externalLibPath.length() != 0) {
+            libraryManagers.add(((ICcApplicationContext) cc.getApplicationContext()).getLibraryManager());
+            for (NodeControllerService nc : ncs) {
+                INcApplicationContext runtimeCtx = (INcApplicationContext) nc.getApplicationContext();
+                libraryManagers.add(runtimeCtx.getLibraryManager());
+            }
+            librarian.install(System.getProperty("external.lib.dataverse", "test"),
+                    System.getProperty("external.lib.libname", "testlib"), externalLibPath);
+        }
+    }
+
     public ClusterControllerService getClusterControllerService() {
         return cc;
     }
@@ -271,14 +290,15 @@
     public static void main(String[] args) throws Exception {
         AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil();
         try {
-            integrationUtil.run(Boolean.getBoolean("cleanup.start"), Boolean.getBoolean("cleanup.shutdown"));
+            integrationUtil.run(Boolean.getBoolean("cleanup.start"), Boolean.getBoolean("cleanup.shutdown"),
+                    System.getProperty("external.lib", ""));
         } catch (Exception e) {
             LOGGER.log(Level.WARNING, "Unexpected exception", e);
             System.exit(1);
         }
     }
 
-    protected void run(boolean cleanupOnStart, boolean cleanupOnShutdown) throws Exception {
+    protected void run(boolean cleanupOnStart, boolean cleanupOnShutdown, String loadExternalLibs) throws Exception {
         Runtime.getRuntime().addShutdownHook(new Thread() {
             @Override
             public void run() {
@@ -291,7 +311,7 @@
         });
         System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, "asterix-build-configuration.xml");
 
-        init(cleanupOnStart);
+        init(cleanupOnStart, loadExternalLibs);
         while (true) {
             Thread.sleep(10000);
         }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
index 5f86c28..4271d55 100755
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
@@ -346,7 +346,7 @@
         if (jarsInLibDir.length > 1) {
             throw new Exception("Incorrect library structure: found multiple library jars");
         }
-        if (jarsInLibDir.length < 0) {
+        if (jarsInLibDir.length <= 0) {
             throw new Exception("Incorrect library structure: could not find library jar");
         }
 
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
similarity index 95%
rename from asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
rename to asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
index 76fa308..b1d2159 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
@@ -28,17 +28,16 @@
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.library.ILibraryManager;
 import org.apache.asterix.event.service.AsterixEventServiceUtil;
-import org.apache.asterix.test.common.ITestLibrarian;
 import org.apache.commons.io.FileUtils;
 import org.apache.hyracks.algebricks.common.utils.Pair;
 
-public class TestLibrarian implements ITestLibrarian {
+public class ExternalUDFLibrarian implements IExternalUDFLibrarian {
 
     // The following list includes a library manager for the CC
     // and library managers for NCs (one-per-NC).
     private final List<ILibraryManager> libraryManagers;
 
-    public TestLibrarian(List<ILibraryManager> libraryManagers) {
+    public ExternalUDFLibrarian(List<ILibraryManager> libraryManagers) {
         this.libraryManagers = libraryManagers;
     }
 
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ITestLibrarian.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java
similarity index 93%
rename from asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ITestLibrarian.java
rename to asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java
index d661710..9a17444 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ITestLibrarian.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.test.common;
+package org.apache.asterix.app.external;
 
 import java.io.IOException;
 import java.rmi.RemoteException;
@@ -24,7 +24,7 @@
 import org.apache.asterix.common.exceptions.ACIDException;
 import org.apache.asterix.common.exceptions.AsterixException;
 
-public interface ITestLibrarian {
+public interface IExternalUDFLibrarian {
     public void install(String dvName, String libName, String libPath) throws IOException, Exception;
 
     public void uninstall(String dvName, String libName) throws RemoteException, AsterixException, ACIDException;
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
index b155b51..1810517 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
@@ -27,7 +27,7 @@
 import java.util.Map;
 import java.util.logging.Logger;
 
-import org.apache.asterix.app.external.TestLibrarian;
+import org.apache.asterix.app.external.ExternalUDFLibrarian;
 import org.apache.asterix.app.nc.NCAppRuntimeContext;
 import org.apache.asterix.app.nc.TransactionSubsystem;
 import org.apache.asterix.common.config.DatasetConfig.IndexType;
@@ -137,7 +137,7 @@
             File outdir = new File(PATH_ACTUAL);
             outdir.mkdirs();
             // remove library directory
-            TestLibrarian.removeLibraryDir();
+            ExternalUDFLibrarian.removeLibraryDir();
             ExecutionTestUtil.setUp(cleanupOnStart,
                     testConfigFileName == null ? TEST_CONFIG_FILE_NAME : testConfigFileName,
                     ExecutionTestUtil.integrationUtil, runHDFS);
@@ -148,7 +148,7 @@
     }
 
     public void deInit() throws Exception {
-        TestLibrarian.removeLibraryDir();
+        ExternalUDFLibrarian.removeLibraryDir();
         ExecutionTestUtil.tearDown(cleanupOnStop);
     }
 
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 4ca9131..0d66256 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -58,6 +58,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.asterix.app.external.IExternalUDFLibrarian;
 import org.apache.asterix.common.api.Duration;
 import org.apache.asterix.common.config.GlobalConfig;
 import org.apache.asterix.common.utils.Servlets;
@@ -134,7 +135,7 @@
      */
     protected final List<InetSocketAddress> endpoints;
     protected int endpointSelector;
-    protected ITestLibrarian librarian;
+    protected IExternalUDFLibrarian librarian;
     private Map<File, TestLoop> testLoops = new HashMap<>();
 
     public TestExecutor() {
@@ -153,7 +154,7 @@
         this.endpoints = endpoints;
     }
 
-    public void setLibrarian(ITestLibrarian librarian) {
+    public void setLibrarian(IExternalUDFLibrarian librarian) {
         this.librarian = librarian;
     }
 
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
index d80eabc..78bed6a 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
@@ -31,7 +31,7 @@
 import java.util.Collection;
 import java.util.List;
 
-import org.apache.asterix.app.external.TestLibrarian;
+import org.apache.asterix.app.external.ExternalUDFLibrarian;
 import org.apache.asterix.common.config.ClusterProperties;
 import org.apache.asterix.common.library.ILibraryManager;
 import org.apache.asterix.test.common.TestExecutor;
@@ -56,7 +56,7 @@
     private static final List<String> badTestCases = new ArrayList<>();
     private static TestExecutor testExecutor;
 
-    private static TestLibrarian librarian;
+    private static ExternalUDFLibrarian librarian;
     private static final int repeat = Integer.getInteger("test.repeat", 1);
 
     public static void setUp(String configFile, TestExecutor executor) throws Exception {
@@ -64,8 +64,8 @@
         File outdir = new File(PATH_ACTUAL);
         outdir.mkdirs();
         List<ILibraryManager> libraryManagers = ExecutionTestUtil.setUp(cleanupOnStart, configFile);
-        TestLibrarian.removeLibraryDir();
-        librarian = new TestLibrarian(libraryManagers);
+        ExternalUDFLibrarian.removeLibraryDir();
+        librarian = new ExternalUDFLibrarian(libraryManagers);
         testExecutor.setLibrarian(librarian);
         if (repeat != 1) {
             System.out.println("FYI: each test will be run " + repeat + " times.");
@@ -79,7 +79,7 @@
             // Check whether there are leaked threads.
             checkThreadLeaks();
         } finally {
-            TestLibrarian.removeLibraryDir();
+            ExternalUDFLibrarian.removeLibraryDir();
             ExecutionTestUtil.tearDown(cleanupOnStop);
             ExecutionTestUtil.integrationUtil.removeTestStorageFiles();
             if (!badTestCases.isEmpty()) {