[NO ISSUE][*DB] Enable loading external UDF for AsterixHyracksIntegrationUtil
- user model changes: no
- storage format changes: no
- interface changes: no
Add external UDF support for AsterixHyracksIntegrationUtil. Developer
may use following system properties to install external UDF for local
environment.
"external.lib" : udf package path.
"external.lib.dataverse": dataverse name (default: test).
"external.lib.libname": library name (default: testlib).
Change-Id: I333238b944b1c6d48a2b80b87a25f600522e0cc4
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2078
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
index 50c3ff6..2a8a831 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
@@ -29,11 +29,14 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.asterix.app.external.ExternalUDFLibrarian;
import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
+import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.config.PropertiesAccessor;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.hyracks.bootstrap.CCApplication;
import org.apache.asterix.hyracks.bootstrap.NCApplication;
import org.apache.commons.io.FileUtils;
@@ -91,7 +94,7 @@
ncApplication.registerConfig(ncConfigManager);
nodeControllers.add(
new NodeControllerService(fixupIODevices(createNCConfig(nodeId, ncConfigManager)), ncApplication));
- } ;
+ }
cc.start();
@@ -122,6 +125,22 @@
this.ncs = nodeControllers.toArray(new NodeControllerService[nodeControllers.size()]);
}
+ public void init(boolean deleteOldInstanceData, String externalLibPath) throws Exception {
+ List<ILibraryManager> libraryManagers = new ArrayList<>();
+ ExternalUDFLibrarian librarian = new ExternalUDFLibrarian(libraryManagers);
+ librarian.cleanup();
+ init(deleteOldInstanceData);
+ if (externalLibPath != null && externalLibPath.length() != 0) {
+ libraryManagers.add(((ICcApplicationContext) cc.getApplicationContext()).getLibraryManager());
+ for (NodeControllerService nc : ncs) {
+ INcApplicationContext runtimeCtx = (INcApplicationContext) nc.getApplicationContext();
+ libraryManagers.add(runtimeCtx.getLibraryManager());
+ }
+ librarian.install(System.getProperty("external.lib.dataverse", "test"),
+ System.getProperty("external.lib.libname", "testlib"), externalLibPath);
+ }
+ }
+
public ClusterControllerService getClusterControllerService() {
return cc;
}
@@ -271,14 +290,15 @@
public static void main(String[] args) throws Exception {
AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil();
try {
- integrationUtil.run(Boolean.getBoolean("cleanup.start"), Boolean.getBoolean("cleanup.shutdown"));
+ integrationUtil.run(Boolean.getBoolean("cleanup.start"), Boolean.getBoolean("cleanup.shutdown"),
+ System.getProperty("external.lib", ""));
} catch (Exception e) {
LOGGER.log(Level.WARNING, "Unexpected exception", e);
System.exit(1);
}
}
- protected void run(boolean cleanupOnStart, boolean cleanupOnShutdown) throws Exception {
+ protected void run(boolean cleanupOnStart, boolean cleanupOnShutdown, String loadExternalLibs) throws Exception {
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
@@ -291,7 +311,7 @@
});
System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, "asterix-build-configuration.xml");
- init(cleanupOnStart);
+ init(cleanupOnStart, loadExternalLibs);
while (true) {
Thread.sleep(10000);
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
index 5f86c28..4271d55 100755
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
@@ -346,7 +346,7 @@
if (jarsInLibDir.length > 1) {
throw new Exception("Incorrect library structure: found multiple library jars");
}
- if (jarsInLibDir.length < 0) {
+ if (jarsInLibDir.length <= 0) {
throw new Exception("Incorrect library structure: could not find library jar");
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
similarity index 95%
rename from asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
rename to asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
index 76fa308..b1d2159 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
@@ -28,17 +28,16 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.event.service.AsterixEventServiceUtil;
-import org.apache.asterix.test.common.ITestLibrarian;
import org.apache.commons.io.FileUtils;
import org.apache.hyracks.algebricks.common.utils.Pair;
-public class TestLibrarian implements ITestLibrarian {
+public class ExternalUDFLibrarian implements IExternalUDFLibrarian {
// The following list includes a library manager for the CC
// and library managers for NCs (one-per-NC).
private final List<ILibraryManager> libraryManagers;
- public TestLibrarian(List<ILibraryManager> libraryManagers) {
+ public ExternalUDFLibrarian(List<ILibraryManager> libraryManagers) {
this.libraryManagers = libraryManagers;
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ITestLibrarian.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java
similarity index 93%
rename from asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ITestLibrarian.java
rename to asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java
index d661710..9a17444 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ITestLibrarian.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.test.common;
+package org.apache.asterix.app.external;
import java.io.IOException;
import java.rmi.RemoteException;
@@ -24,7 +24,7 @@
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
-public interface ITestLibrarian {
+public interface IExternalUDFLibrarian {
public void install(String dvName, String libName, String libPath) throws IOException, Exception;
public void uninstall(String dvName, String libName) throws RemoteException, AsterixException, ACIDException;
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
index b155b51..1810517 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
@@ -27,7 +27,7 @@
import java.util.Map;
import java.util.logging.Logger;
-import org.apache.asterix.app.external.TestLibrarian;
+import org.apache.asterix.app.external.ExternalUDFLibrarian;
import org.apache.asterix.app.nc.NCAppRuntimeContext;
import org.apache.asterix.app.nc.TransactionSubsystem;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
@@ -137,7 +137,7 @@
File outdir = new File(PATH_ACTUAL);
outdir.mkdirs();
// remove library directory
- TestLibrarian.removeLibraryDir();
+ ExternalUDFLibrarian.removeLibraryDir();
ExecutionTestUtil.setUp(cleanupOnStart,
testConfigFileName == null ? TEST_CONFIG_FILE_NAME : testConfigFileName,
ExecutionTestUtil.integrationUtil, runHDFS);
@@ -148,7 +148,7 @@
}
public void deInit() throws Exception {
- TestLibrarian.removeLibraryDir();
+ ExternalUDFLibrarian.removeLibraryDir();
ExecutionTestUtil.tearDown(cleanupOnStop);
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 4ca9131..0d66256 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -58,6 +58,7 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.asterix.app.external.IExternalUDFLibrarian;
import org.apache.asterix.common.api.Duration;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.utils.Servlets;
@@ -134,7 +135,7 @@
*/
protected final List<InetSocketAddress> endpoints;
protected int endpointSelector;
- protected ITestLibrarian librarian;
+ protected IExternalUDFLibrarian librarian;
private Map<File, TestLoop> testLoops = new HashMap<>();
public TestExecutor() {
@@ -153,7 +154,7 @@
this.endpoints = endpoints;
}
- public void setLibrarian(ITestLibrarian librarian) {
+ public void setLibrarian(IExternalUDFLibrarian librarian) {
this.librarian = librarian;
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
index d80eabc..78bed6a 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
@@ -31,7 +31,7 @@
import java.util.Collection;
import java.util.List;
-import org.apache.asterix.app.external.TestLibrarian;
+import org.apache.asterix.app.external.ExternalUDFLibrarian;
import org.apache.asterix.common.config.ClusterProperties;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.test.common.TestExecutor;
@@ -56,7 +56,7 @@
private static final List<String> badTestCases = new ArrayList<>();
private static TestExecutor testExecutor;
- private static TestLibrarian librarian;
+ private static ExternalUDFLibrarian librarian;
private static final int repeat = Integer.getInteger("test.repeat", 1);
public static void setUp(String configFile, TestExecutor executor) throws Exception {
@@ -64,8 +64,8 @@
File outdir = new File(PATH_ACTUAL);
outdir.mkdirs();
List<ILibraryManager> libraryManagers = ExecutionTestUtil.setUp(cleanupOnStart, configFile);
- TestLibrarian.removeLibraryDir();
- librarian = new TestLibrarian(libraryManagers);
+ ExternalUDFLibrarian.removeLibraryDir();
+ librarian = new ExternalUDFLibrarian(libraryManagers);
testExecutor.setLibrarian(librarian);
if (repeat != 1) {
System.out.println("FYI: each test will be run " + repeat + " times.");
@@ -79,7 +79,7 @@
// Check whether there are leaked threads.
checkThreadLeaks();
} finally {
- TestLibrarian.removeLibraryDir();
+ ExternalUDFLibrarian.removeLibraryDir();
ExecutionTestUtil.tearDown(cleanupOnStop);
ExecutionTestUtil.integrationUtil.removeTestStorageFiles();
if (!badTestCases.isEmpty()) {