[ASTERIXDB-2176] Python UDFs
- user model changes: yes
- storage format changes: no
- interface changes: yes
Details:
- Allow UDFs to be shiv-packaged python modules with their
dependencies
- Use pyro for python RPC
- Maven build for pyro package into server
- Remove JObject spatial types
- Simpler conversion from java primitive/standard types and
collections
to JObject equivalents
Change-Id: Ibea23a2e9308132f343d80eff04ede9a235aa021
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/5526
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index a34a991..2b703a2 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@ -31,6 +31,7 @@
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.WarningCollector;
import org.apache.asterix.common.exceptions.WarningUtil;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.dataflow.data.common.ExpressionTypeComputer;
import org.apache.asterix.dataflow.data.nontagged.MissingWriterFactory;
import org.apache.asterix.formats.nontagged.ADMPrinterFactoryProvider;
@@ -47,7 +48,6 @@
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.constants.AsterixConstantValue;
import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.ExternalFunctionLanguage;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
import org.apache.asterix.om.types.ARecordType;
diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml
index 392dbd8..e5e978a 100644
--- a/asterixdb/asterix-app/pom.xml
+++ b/asterixdb/asterix-app/pom.xml
@@ -37,6 +37,8 @@
<root.dir>${basedir}/..</root.dir>
<appendedResourcesDirectory>${basedir}/src/main/appended-resources</appendedResourcesDirectory>
<sonar.sources>pom.xml,src/main/java,src/main/resources</sonar.sources>
+ <pip.path>${project.build.directory}${file.separator}bin${file.separator}pip3</pip.path>
+ <shiv.path>${project.build.directory}${file.separator}bin${file.separator}shiv</shiv.path>
</properties>
<build>
<plugins>
@@ -163,6 +165,93 @@
</configuration>
</plugin>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>venv</id>
+ <phase>${pyro-shim.stage}</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <!--suppress UnresolvedMavenProperty -->
+ <executable>${python.path}</executable>
+ <workingDirectory>${project.build.directory}</workingDirectory>
+ <arguments>
+ <argument>-m</argument>
+ <argument>venv</argument>
+ <argument>${project.build.directory}</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>shiv-install</id>
+ <phase>${pyro-shim.stage}</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>${pip.path}</executable>
+ <workingDirectory>${project.build.directory}</workingDirectory>
+ <arguments>
+ <argument>install</argument>
+ <argument>--exists-action</argument>
+ <argument>w</argument>
+ <argument>--upgrade</argument>
+ <argument>shiv</argument>
+ </arguments>
+ <environmentVariables>
+ <VIRTUALENV>${project.build.directory}</VIRTUALENV>
+ <PATH>${project.build.directory}${path.separator}${env.PATH}${file.separator}bin</PATH>
+ </environmentVariables>
+ </configuration>
+ </execution>
+ <execution>
+ <id>shiv-pyro-shim</id>
+ <phase>${pyro-shim.stage}</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>${shiv.path}</executable>
+ <workingDirectory>${project.build.directory}</workingDirectory>
+ <arguments>
+ <argument>-o </argument>
+ <argument>${project.build.directory}${file.separator}classes${file.separator}pyro4.pyz</argument>
+ <argument>pyro4</argument>
+ </arguments>
+ <environmentVariables>
+ <VIRTUALENV>${project.build.directory}</VIRTUALENV>
+ <PATH>${project.build.directory}${path.separator}${env.PATH}${file.separator}bin</PATH>
+ </environmentVariables>
+ </configuration>
+ </execution>
+ <execution>
+ <id>shiv-test-lib</id>
+ <phase>${pytestlib.stage}</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>${shiv.path}</executable>
+ <workingDirectory>${project.build.directory}</workingDirectory>
+ <arguments>
+ <argument>-o </argument>
+ <argument>${project.build.directory}${file.separator}TweetSent.pyz</argument>
+ <argument>--site-packages</argument>
+ <argument>${project.build.directory}${file.separator}..${file.separator}src${file.separator}test${file.separator}resources${file.separator}TweetSent</argument>
+ <argument>scikit-learn</argument>
+ </arguments>
+ <environmentVariables>
+ <VIRTUALENV>${project.build.directory}</VIRTUALENV>
+ <PATH>${project.build.directory}${path.separator}${env.PATH}${file.separator}bin</PATH>
+ </environmentVariables>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<executions>
@@ -286,6 +375,18 @@
</build>
</profile>
<profile>
+ <id>windows.python.envs</id>
+ <activation>
+ <os>
+ <family>Windows</family>
+ </os>
+ </activation>
+ <properties>
+ <pip.path>${project.build.directory}${file.separator}Scripts${file.separator}pip3.exe</pip.path>
+ <shiv.path>${project.build.directory}${file.separator}Scripts${file.separator}shiv.exe</shiv.path>
+ </properties>
+ </profile>
+ <profile>
<id>asterix-gerrit-asterix-app</id>
<properties>
<test.excludes>**/SqlppExecutionWithCancellationTest.java,**/DmlTest.java,**/RepeatedTest.java,**/SqlppExecutionTest.java,**/AqlExecutionTest.java,**/*Compression*Test.java,**/*Ssl*Test.java</test.excludes>
@@ -322,7 +423,7 @@
<id>asterix-gerrit-asterix-app-sql-execution</id>
<properties>
<test.excludes>**/*.java</test.excludes>
- <itest.includes>**/SqlppExecution*IT.java</itest.includes>
+ <itest.includes>**/SqlppExecution*IT.java,**/ExternalPythonFunction*IT.java</itest.includes>
<failIfNoTests>false</failIfNoTests>
</properties>
</profile>
@@ -346,7 +447,7 @@
<id>asterix-gerrit-verify-asterix-app</id>
<properties>
<test.includes>**/AqlExecutionTest.java</test.includes>
- <itest.excludes>**/SqlppExecution*IT.java,**/SqlppRQG*IT.java,**/RebalanceWithCancellationIT.java</itest.excludes>
+ <itest.excludes>**/External*IT.java,**/SqlppExecution*IT.java,**/SqlppRQG*IT.java,**/RebalanceWithCancellationIT.java</itest.excludes>
<failIfNoTests>false</failIfNoTests>
</properties>
</profile>
@@ -726,5 +827,10 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/UdfApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/UdfApiServlet.java
index 0f5d542..71be106 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/UdfApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/UdfApiServlet.java
@@ -18,15 +18,19 @@
*/
package org.apache.asterix.api.http.server;
+import static org.apache.asterix.common.functions.ExternalFunctionLanguage.JAVA;
+import static org.apache.asterix.common.functions.ExternalFunctionLanguage.PYTHON;
+import static org.apache.asterix.common.library.LibraryDescriptor.DESCRIPTOR_NAME;
+
import java.io.File;
+import java.io.IOException;
import java.io.PrintWriter;
-import java.io.RandomAccessFile;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
+import java.nio.file.Paths;
import java.rmi.RemoteException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentMap;
import org.apache.asterix.app.external.ExternalLibraryUtils;
@@ -34,6 +38,10 @@
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+import org.apache.asterix.common.library.ILibrary;
+import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.common.library.LibraryDescriptor;
import org.apache.asterix.common.messaging.api.ICCMessageBroker;
import org.apache.asterix.common.messaging.api.INcAddressedMessage;
import org.apache.asterix.common.metadata.DataverseName;
@@ -50,25 +58,32 @@
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.deployment.DeploymentId;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+import org.apache.hyracks.control.common.deployment.DeploymentUtils;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.util.file.FileUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import io.netty.buffer.ByteBuf;
+import com.google.common.collect.ImmutableMap;
+
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.HttpResponseStatus;
import io.netty.handler.codec.http.QueryStringDecoder;
+import io.netty.handler.codec.http.multipart.FileUpload;
+import io.netty.handler.codec.http.multipart.HttpPostRequestDecoder;
+import io.netty.handler.codec.http.multipart.InterfaceHttpData;
public class UdfApiServlet extends BasicAuthServlet {
private static final Logger LOGGER = LogManager.getLogger();
private final ICcApplicationContext appCtx;
private final ICCMessageBroker broker;
- public static final String UDF_TMP_DIR_PREFIX = "udf_temp";
+ private static final String UDF_TMP_DIR_PREFIX = "udf_temp";
public static final int UDF_RESPONSE_TIMEOUT = 5000;
- public static final int URL_PREFIX_LENGTH = 3;
+ private Map<String, ExternalFunctionLanguage> exensionMap =
+ new ImmutableMap.Builder<String, ExternalFunctionLanguage>().put("pyz", PYTHON).put("zip", JAVA).build();
public UdfApiServlet(ICcApplicationContext appCtx, ConcurrentMap<String, Object> ctx, String... paths) {
super(ctx, paths);
@@ -88,7 +103,6 @@
@Override
protected void post(IServletRequest request, IServletResponse response) {
-
PrintWriter responseWriter = response.writer();
FullHttpRequest req = request.getHttpRequest();
Pair<String, DataverseName> resourceNames;
@@ -100,11 +114,21 @@
}
String resourceName = resourceNames.first;
DataverseName dataverse = resourceNames.second;
+ HttpPostRequestDecoder multipartDec = new HttpPostRequestDecoder(req);
+ File udfFile = null;
IMetadataLockUtil mdLockUtil = appCtx.getMetadataLockUtil();
MetadataTransactionContext mdTxnCtx = null;
LockList mdLockList = null;
- File udf = null;
try {
+ if (!multipartDec.hasNext() || multipartDec.getBodyHttpDatas().size() != 1) {
+ response.setStatus(HttpResponseStatus.BAD_REQUEST);
+ return;
+ }
+ InterfaceHttpData f = multipartDec.getBodyHttpDatas().get(0);
+ if (!f.getHttpDataType().equals(InterfaceHttpData.HttpDataType.FileUpload)) {
+ response.setStatus(HttpResponseStatus.BAD_REQUEST);
+ return;
+ }
MetadataManager.INSTANCE.init();
mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
MetadataProvider metadataProvider = MetadataProvider.create(appCtx, null);
@@ -116,29 +140,27 @@
if (!workingDir.exists()) {
FileUtil.forceMkdirs(workingDir);
}
- udf = File.createTempFile(resourceName, ".zip", workingDir);
- try (RandomAccessFile raf = new RandomAccessFile(udf, "rw")) {
- ByteBuf reqContent = req.content();
- raf.setLength(reqContent.readableBytes());
- FileChannel fc = raf.getChannel();
- ByteBuffer content = reqContent.nioBuffer();
- while (content.hasRemaining()) {
- fc.write(content);
- }
+ FileUpload udf = (FileUpload) f;
+ String[] fileNameParts = udf.getFilename().split("\\.");
+ String suffix = fileNameParts[fileNameParts.length - 1];
+ ExternalFunctionLanguage libLang = exensionMap.get(suffix);
+ if (libLang == null) {
+ response.setStatus(HttpResponseStatus.BAD_REQUEST);
+ return;
}
- setupBinariesAndClassloaders(dataverse, resourceName, udf);
+ LibraryDescriptor desc = new LibraryDescriptor(libLang);
+ udfFile = File.createTempFile(resourceName, "." + suffix, workingDir);
+ udf.renameTo(udfFile);
+ setupBinariesAndClassloaders(dataverse, resourceName, udfFile, desc);
installLibrary(mdTxnCtx, dataverse, resourceName);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ response.setStatus(HttpResponseStatus.OK);
} catch (Exception e) {
try {
ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverse, resourceName);
} catch (Exception e2) {
e.addSuppressed(e2);
}
- response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
- responseWriter.write(e.getMessage());
- responseWriter.flush();
- LOGGER.error(e);
if (mdTxnCtx != null) {
try {
MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
@@ -146,33 +168,47 @@
LOGGER.error("Unable to abort metadata transaction", r);
}
}
- if (udf != null) {
- udf.delete();
- }
- return;
+ response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
+ responseWriter.write(e.getMessage());
+ responseWriter.flush();
+ LOGGER.error(e);
} finally {
+ multipartDec.destroy();
+ if (udfFile != null) {
+ udfFile.delete();
+ }
if (mdLockList != null) {
mdLockList.unlock();
}
}
- response.setStatus(HttpResponseStatus.OK);
-
}
- private void setupBinariesAndClassloaders(DataverseName dataverse, String resourceName, File udf) throws Exception {
+ private File writeDescriptor(File folder, LibraryDescriptor desc) throws IOException {
+ IPersistedResourceRegistry reg = appCtx.getServiceContext().getPersistedResourceRegistry();
+ byte[] bytes = OBJECT_MAPPER.writeValueAsBytes(desc.toJson(reg));
+ File descFile = new File(folder, DESCRIPTOR_NAME);
+ FileUtil.writeAndForce(Paths.get(descFile.getAbsolutePath()), bytes);
+ return descFile;
+ }
+
+ private void setupBinariesAndClassloaders(DataverseName dataverse, String resourceName, File udfFile,
+ LibraryDescriptor desc) throws Exception {
IHyracksClientConnection hcc = appCtx.getHcc();
- DeploymentId udfName = new DeploymentId(makeDeploymentId(dataverse, resourceName));
- ClassLoader cl = appCtx.getLibraryManager().getLibraryClassLoader(dataverse, resourceName);
- if (cl != null) {
- //prepare to replace the binary
- ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverse, resourceName);
+ ILibraryManager libMgr = appCtx.getLibraryManager();
+ DeploymentId udfName = new DeploymentId(ExternalLibraryUtils.makeDeploymentId(dataverse, resourceName));
+ ILibrary lib = libMgr.getLibrary(dataverse, resourceName);
+ if (lib != null) {
+ deleteUdf(dataverse, resourceName);
}
- hcc.deployBinary(udfName, Arrays.asList(udf.toString()), true);
- //setup for CC
- ExternalLibraryUtils.setUpExternaLibrary(appCtx.getLibraryManager(),
+ File descriptor = writeDescriptor(udfFile.getParentFile(), desc);
+ hcc.deployBinary(udfName, Arrays.asList(udfFile.getAbsolutePath(), descriptor.getAbsolutePath()), true);
+ String deployedPath =
FileUtil.joinPath(appCtx.getServiceContext().getServerCtx().getBaseDir().getAbsolutePath(),
- "applications", udfName.toString()));
- //setup NCs
+ DeploymentUtils.DEPLOYMENT, udfName.toString());
+ if (!descriptor.delete()) {
+ throw new IOException("Could not remove already uploaded library descriptor");
+ }
+ libMgr.setUpDeployedLibrary(deployedPath);
long reqId = broker.newRequestId();
List<INcAddressedMessage> requests = new ArrayList<>();
List<String> ncs = new ArrayList<>(appCtx.getClusterStateManager().getParticipantNodes());
@@ -182,12 +218,11 @@
private static void installLibrary(MetadataTransactionContext mdTxnCtx, DataverseName dataverse, String libraryName)
throws RemoteException, AlgebricksException {
- Library libraryInMetadata = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName);
- // Get the dataverse
Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse);
if (dv == null) {
throw new AsterixException(ErrorCode.UNKNOWN_DATAVERSE);
}
+ Library libraryInMetadata = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName);
if (libraryInMetadata != null) {
//replacing binary, library already exists
return;
@@ -224,13 +259,6 @@
MetadataManager.INSTANCE.dropLibrary(mdTxnCtx, dataverse, libraryName);
}
- public static String makeDeploymentId(DataverseName dv, String resourceName) {
- List<String> dvParts = dv.getParts();
- dvParts.add(resourceName);
- DataverseName dvWithLibrarySuffix = DataverseName.create(dvParts);
- return dvWithLibrarySuffix.getCanonicalForm();
- }
-
@Override
protected void delete(IServletRequest request, IServletResponse response) {
Pair<String, DataverseName> resourceNames;
@@ -243,6 +271,18 @@
PrintWriter responseWriter = response.writer();
String resourceName = resourceNames.first;
DataverseName dataverse = resourceNames.second;
+ try {
+ deleteUdf(dataverse, resourceName);
+ } catch (Exception e) {
+ response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
+ responseWriter.write(e.getMessage());
+ responseWriter.flush();
+ return;
+ }
+ response.setStatus(HttpResponseStatus.OK);
+ }
+
+ private void deleteUdf(DataverseName dataverse, String resourceName) throws Exception {
IMetadataLockUtil mdLockUtil = appCtx.getMetadataLockUtil();
MetadataTransactionContext mdTxnCtx = null;
LockList mdLockList = null;
@@ -262,16 +302,12 @@
} catch (RemoteException r) {
LOGGER.error("Unable to abort metadata transaction", r);
}
- response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
- responseWriter.write(e.getMessage());
- responseWriter.flush();
LOGGER.error(e);
- return;
+ throw e;
} finally {
if (mdLockList != null) {
mdLockList.unlock();
}
}
- response.setStatus(HttpResponseStatus.OK);
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
index 47a460e..e5d874d 100755
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
@@ -19,57 +19,22 @@
package org.apache.asterix.app.external;
import static org.apache.asterix.api.http.server.UdfApiServlet.UDF_RESPONSE_TIMEOUT;
-import static org.apache.asterix.api.http.server.UdfApiServlet.makeDeploymentId;
-import java.io.File;
-import java.io.FilenameFilter;
-import java.net.URL;
-import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.List;
import org.apache.asterix.app.message.DeleteUdfMessage;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
-import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.messaging.api.ICCMessageBroker;
import org.apache.asterix.common.messaging.api.INcAddressedMessage;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.hyracks.api.deployment.DeploymentId;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
public class ExternalLibraryUtils {
- private static final Logger LOGGER = LogManager.getLogger();
- private static final FilenameFilter nonHiddenFileNameFilter = (dir, name) -> !name.startsWith(".");
-
private ExternalLibraryUtils() {
}
- public static void setUpExternaLibrary(ILibraryManager externalLibraryManager, String libraryPath)
- throws Exception {
- // get the installed library dirs
- String[] parts = libraryPath.split(File.separator);
- DataverseName catenatedDv = DataverseName.createFromCanonicalForm(parts[parts.length - 1]);
- String libraryName = catenatedDv.getParts().get(catenatedDv.getParts().size() - 1);
- DataverseName dvName = DataverseName.create(catenatedDv.getParts(), 0, catenatedDv.getParts().size() - 1);
- registerClassLoader(externalLibraryManager, dvName, libraryName, libraryPath);
- }
-
- public static void setUpInstalledLibraries(ILibraryManager externalLibraryManager, File appDir) throws Exception {
- File[] libs = appDir.listFiles(new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return dir.isDirectory();
- }
- });
- if (libs != null) {
- for (File lib : libs) {
- setUpExternaLibrary(externalLibraryManager, lib.getAbsolutePath());
- }
- }
- }
-
public static void deleteDeployedUdf(ICCMessageBroker broker, ICcApplicationContext appCtx,
DataverseName dataverseName, String lib) throws Exception {
long reqId = broker.newRequestId();
@@ -77,107 +42,14 @@
List<String> ncs = new ArrayList<>(appCtx.getClusterStateManager().getParticipantNodes());
ncs.forEach(s -> requests.add(new DeleteUdfMessage(dataverseName, lib, reqId)));
broker.sendSyncRequestToNCs(reqId, ncs, requests, UDF_RESPONSE_TIMEOUT);
- appCtx.getLibraryManager().deregisterLibraryClassLoader(dataverseName, lib);
+ appCtx.getLibraryManager().deregister(dataverseName, lib);
appCtx.getHcc().unDeployBinary(new DeploymentId(makeDeploymentId(dataverseName, lib)));
}
- /**
- * register the library class loader with the external library manager
- *
- * @param dataverse
- * @param libraryPath
- * @throws Exception
- */
- protected static void registerClassLoader(ILibraryManager externalLibraryManager, DataverseName dataverse,
- String name, String libraryPath) throws Exception {
- // get the class loader
- URLClassLoader classLoader = getLibraryClassLoader(dataverse, name, libraryPath);
- // register it with the external library manager
- externalLibraryManager.registerLibraryClassLoader(dataverse, name, classLoader);
+ public static String makeDeploymentId(DataverseName dv, String resourceName) {
+ List<String> dvParts = dv.getParts();
+ dvParts.add(resourceName);
+ DataverseName dvWithLibrarySuffix = DataverseName.create(dvParts);
+ return dvWithLibrarySuffix.getCanonicalForm();
}
-
- /**
- * Get the class loader for the library
- *
- * @param dataverse
- * @param libraryPath
- * @return
- * @throws Exception
- */
- private static URLClassLoader getLibraryClassLoader(DataverseName dataverse, String name, String libraryPath)
- throws Exception {
- // Get a reference to the library directory
- File installDir = new File(libraryPath);
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("Installing lirbary " + name + " in dataverse " + dataverse + "." + " Install Directory: "
- + installDir.getAbsolutePath());
- }
-
- // get a reference to the specific library dir
- File libDir = installDir;
-
- FilenameFilter jarFileFilter = new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return name.endsWith(".jar");
- }
- };
-
- // Get the jar file <Allow only a single jar file>
- String[] jarsInLibDir = libDir.list(jarFileFilter);
- if (jarsInLibDir.length > 1) {
- throw new Exception("Incorrect library structure: found multiple library jars");
- }
- if (jarsInLibDir.length <= 0) {
- throw new Exception("Incorrect library structure: could not find library jar");
- }
-
- File libJar = new File(libDir, jarsInLibDir[0]);
- // get the jar dependencies
- File libDependencyDir = new File(libDir.getAbsolutePath() + File.separator + "lib");
- int numDependencies = 1;
- String[] libraryDependencies = null;
- if (libDependencyDir.exists()) {
- libraryDependencies = libDependencyDir.list(jarFileFilter);
- numDependencies += libraryDependencies.length;
- }
-
- ClassLoader parentClassLoader = ExternalLibraryUtils.class.getClassLoader();
- URL[] urls = new URL[numDependencies];
- int count = 0;
- // get url of library
- urls[count++] = libJar.toURI().toURL();
-
- // get urls for dependencies
- if (libraryDependencies != null && libraryDependencies.length > 0) {
- for (String dependency : libraryDependencies) {
- File file = new File(libDependencyDir + File.separator + dependency);
- urls[count++] = file.toURI().toURL();
- }
- }
-
- if (LOGGER.isInfoEnabled()) {
- StringBuilder logMesg = new StringBuilder("Classpath for library " + dataverse + ": ");
- for (URL url : urls) {
- logMesg.append(url.getFile() + File.pathSeparatorChar);
- }
- LOGGER.info(logMesg.toString());
- }
-
- // create and return the class loader
- return new ExternalLibraryClassLoader(urls, parentClassLoader);
- }
-
- /**
- * @return the directory "System.getProperty("app.home", System.getProperty("user.home")/lib/udfs/uninstall"
- */
- protected static File getLibraryUninstallDir() {
- return new File(System.getProperty("app.home", System.getProperty("user.home")) + File.separator + "lib"
- + File.separator + "udfs" + File.separator + "uninstall");
- }
-
- public static String getExternalFunctionFullName(String libraryName, String functionName) {
- return libraryName + "#" + functionName;
- }
-
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
index 531f31c..1168567 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
@@ -32,6 +32,7 @@
import org.apache.asterix.common.dataflow.DatasetLocalResource;
import org.apache.asterix.common.ioopcallbacks.LSMIndexIOOperationCallbackFactory;
import org.apache.asterix.common.ioopcallbacks.LSMIndexPageWriteCallbackFactory;
+import org.apache.asterix.common.library.LibraryDescriptor;
import org.apache.asterix.common.transactions.Checkpoint;
import org.apache.asterix.dataflow.data.common.AListElementTokenFactory;
import org.apache.asterix.dataflow.data.common.AOrderedListBinaryTokenizerFactory;
@@ -289,6 +290,9 @@
//ICompressorDecompressorFactory
CompressionManager.registerCompressorDecompressorsFactoryClasses(registeredClasses);
+
+ //External Libraries
+ registeredClasses.put("LibraryDescriptor", LibraryDescriptor.class);
}
@Override
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DeleteUdfMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DeleteUdfMessage.java
index 7bcfa9b..890f1fe 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DeleteUdfMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DeleteUdfMessage.java
@@ -32,6 +32,6 @@
@Override
protected void handleAction(ILibraryManager mgr, INcApplicationContext appCtx) {
- mgr.deregisterLibraryClassLoader(dataverseName, libraryName);
+ mgr.deregister(dataverseName, libraryName);
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/LoadUdfMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/LoadUdfMessage.java
index 7ccdfd4..f2d9174 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/LoadUdfMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/LoadUdfMessage.java
@@ -18,10 +18,10 @@
*/
package org.apache.asterix.app.message;
-import org.apache.asterix.app.external.ExternalLibraryUtils;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.hyracks.control.common.deployment.DeploymentUtils;
import org.apache.hyracks.util.file.FileUtil;
public class LoadUdfMessage extends AbstractUdfMessage {
@@ -34,8 +34,8 @@
@Override
protected void handleAction(ILibraryManager mgr, INcApplicationContext appCtx) throws Exception {
- ExternalLibraryUtils.setUpExternaLibrary(mgr,
+ appCtx.getLibraryManager().setUpDeployedLibrary(
FileUtil.joinPath(appCtx.getServiceContext().getServerCtx().getBaseDir().getAbsolutePath(),
- "applications", dataverseName.getCanonicalForm() + "." + libraryName)); //TODO(MULTI_PART_DATAVERSE_NAME):REVISIT
+ DeploymentUtils.DEPLOYMENT, dataverseName.getCanonicalForm() + "." + libraryName));
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index 01fa365..3cf57c5 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -166,11 +166,12 @@
replicationProperties = propertiesFactory.newReplicationProperties();
messagingProperties = propertiesFactory.newMessagingProperties();
nodeProperties = propertiesFactory.newNodeProperties();
- libraryManager = new ExternalLibraryManager();
ncExtensionManager = extensionManager;
componentProvider = new StorageComponentProvider();
resourceIdFactory = new GlobalResourceIdFactoryProvider(ncServiceContext).createResourceIdFactory();
persistedResourceRegistry = ncServiceContext.getPersistedResourceRegistry();
+ libraryManager =
+ new ExternalLibraryManager(ncServiceContext.getServerCtx().getAppDir(), persistedResourceRegistry);
cacheManager = new CacheManager();
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/ExternalLibrarySetupTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/ExternalLibrarySetupTask.java
index 40262ac..1656ce5 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/ExternalLibrarySetupTask.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/ExternalLibrarySetupTask.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.app.nc.task;
-import org.apache.asterix.app.external.ExternalLibraryUtils;
import org.apache.asterix.common.api.INCLifecycleTask;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.hyracks.api.control.CcId;
@@ -38,8 +37,7 @@
public void perform(CcId ccId, IControllerService cs) throws HyracksDataException {
INcApplicationContext appContext = (INcApplicationContext) cs.getApplicationContext();
try {
- ExternalLibraryUtils.setUpInstalledLibraries(appContext.getLibraryManager(),
- cs.getContext().getServerCtx().getAppDir());
+ appContext.getLibraryManager().scanLibraries(cs.getContext().getServerCtx().getAppDir());
} catch (Exception e) {
throw HyracksDataException.create(e);
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 814c96e..067ee6e 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -78,6 +78,7 @@
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.common.exceptions.WarningCollector;
import org.apache.asterix.common.exceptions.WarningUtil;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.common.messaging.api.ICCMessageBroker;
import org.apache.asterix.common.metadata.DataverseName;
@@ -169,7 +170,6 @@
import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.metadata.utils.MetadataUtil;
import org.apache.asterix.om.base.IAObject;
-import org.apache.asterix.om.functions.ExternalFunctionLanguage;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.AUnionType;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
index ae85381..bfc8154 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
@@ -56,7 +56,7 @@
import org.apache.asterix.app.active.ActiveNotificationHandler;
import org.apache.asterix.app.cc.CCExtensionManager;
import org.apache.asterix.app.config.ConfigValidator;
-import org.apache.asterix.app.external.ExternalLibraryUtils;
+import org.apache.asterix.app.io.PersistedResourceRegistry;
import org.apache.asterix.app.replication.NcLifecycleCoordinator;
import org.apache.asterix.app.result.JobResultCallback;
import org.apache.asterix.common.api.AsterixThreadFactory;
@@ -143,25 +143,22 @@
final ClusterControllerService controllerService =
(ClusterControllerService) ccServiceCtx.getControllerService();
ccServiceCtx.setMessageBroker(new CCMessageBroker(controllerService));
-
+ ccServiceCtx.setPersistedResourceRegistry(new PersistedResourceRegistry());
configureLoggingLevel(ccServiceCtx.getAppConfig().getLoggingLevel(ExternalProperties.Option.LOG_LEVEL));
-
LOGGER.info("Starting Asterix cluster controller");
-
String strIP = ccServiceCtx.getCCContext().getClusterControllerInfo().getClientNetAddress();
int port = ccServiceCtx.getCCContext().getClusterControllerInfo().getClientNetPort();
hcc = new HyracksConnection(strIP, port,
ccServiceCtx.getControllerService().getNetworkSecurityManager().getSocketChannelFactory());
MetadataBuiltinFunctions.init();
- ILibraryManager libraryManager = new ExternalLibraryManager();
ReplicationProperties repProp =
new ReplicationProperties(PropertiesAccessor.getInstance(ccServiceCtx.getAppConfig()));
INcLifecycleCoordinator lifecycleCoordinator = createNcLifeCycleCoordinator(repProp.isReplicationEnabled());
- ExternalLibraryUtils.setUpInstalledLibraries(libraryManager, ccServiceCtx.getServerCtx().getAppDir());
componentProvider = new StorageComponentProvider();
-
ccExtensionManager = new CCExtensionManager(new ArrayList<>(getExtensions()));
IGlobalRecoveryManager globalRecoveryManager = createGlobalRecoveryManager();
+ ILibraryManager libraryManager = new ExternalLibraryManager(ccServiceCtx.getServerCtx().getAppDir(),
+ ccServiceCtx.getPersistedResourceRegistry());
appCtx = createApplicationContext(libraryManager, globalRecoveryManager, lifecycleCoordinator,
() -> new Receptionist("CC"), ConfigValidator::new, ccExtensionManager);
final CCConfig ccConfig = controllerService.getCCConfig();
diff --git a/asterixdb/asterix-app/src/main/resources/entrypoint.py b/asterixdb/asterix-app/src/main/resources/entrypoint.py
new file mode 100644
index 0000000..cd3298e
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/resources/entrypoint.py
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import math,sys
+sys.path.insert(0,'./site-packages/')
+import Pyro4
+from importlib import import_module
+from pathlib import Path
+
+@Pyro4.expose
+class Wrapper(object):
+ wrapped_module = None
+ wrapped_class = None
+ wrapped_fn = None
+
+ def __init__(self, module_name, class_name, fn_name):
+ self.wrapped_module = import_module(module_name)
+ # do not allow modules to be called that are not part of the uploaded module
+ if not self.check_module_path(self.wrapped_module):
+ wrapped_module = None
+ return None
+ if class_name is not None:
+ self.wrapped_class = getattr(import_module(module_name),class_name)()
+ if self.wrapped_class is not None:
+ self.wrapped_fn = getattr(self.wrapped_class,fn_name)
+ else:
+ self.wrapped_fn = locals()[fn_name]
+
+ def nextTuple(self, *args):
+ return self.wrapped_fn(args)
+
+ def ping(self):
+ return "pong"
+
+ def check_module_path(self,module):
+ cwd = Path('.').resolve()
+ module_path = Path(module.__file__).resolve()
+ return cwd in module_path.parents
+
+
+port = int(sys.argv[1])
+wrap = Wrapper(sys.argv[2],sys.argv[3],sys.argv[4])
+d = Pyro4.Daemon(host="127.0.0.1",port=port)
+d.register(wrap,"nextTuple")
+print(Pyro4.config.dump())
+d.requestLoop()
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
index 0c3ac81..2dc7326 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
@@ -24,6 +24,7 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
@@ -34,7 +35,9 @@
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.entity.FileEntity;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.mime.HttpMultipartMode;
+import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
@@ -48,7 +51,7 @@
private String host;
private int port;
- public ExternalUDFLibrarian(String host, int port) {
+ private ExternalUDFLibrarian(String host, int port) {
hc = new DefaultHttpClient();
this.host = host;
this.port = port;
@@ -71,7 +74,10 @@
AuthCache ac = new BasicAuthCache();
ac.put(h, new BasicScheme());
hcCtx.setAuthCache(ac);
- post.setEntity(new FileEntity(new File(libPath), "application/octet-stream"));
+ File lib = new File(libPath);
+ HttpEntity file = MultipartEntityBuilder.create().setMode(HttpMultipartMode.STRICT)
+ .addBinaryBody("lib", lib, ContentType.DEFAULT_BINARY, lib.getName()).build();
+ post.setEntity(file);
HttpResponse response = hc.execute(post, hcCtx);
response.getEntity().consumeContent();
if (response.getStatusLine().getStatusCode() != 200) {
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExternalPythonFunctionIT.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExternalPythonFunctionIT.java
new file mode 100644
index 0000000..ac2bc6b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExternalPythonFunctionIT.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.asterix.test.runtime;
+
+import java.util.Collection;
+
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Runs the SQLPP runtime tests with the storage parallelism.
+ */
+@RunWith(Parameterized.class)
+public class ExternalPythonFunctionIT {
+ protected static final String TEST_CONFIG_FILE_NAME = "src/test/resources/cc.conf";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME, new TestExecutor());
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ LangExecutionUtil.tearDown();
+ }
+
+ @Parameters(name = "ExternalPythonFunctionIT {index}: {0}")
+ public static Collection<Object[]> tests() throws Exception {
+ return LangExecutionUtil.tests("only_sqlpp.xml", "testsuite_it_python.xml");
+ }
+
+ protected TestCaseContext tcCtx;
+
+ public ExternalPythonFunctionIT(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
+ @Test
+ public void test() throws Exception {
+ LangExecutionUtil.test(tcCtx);
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/resources/TweetSent/sentiment.py b/asterixdb/asterix-app/src/test/resources/TweetSent/sentiment.py
new file mode 100644
index 0000000..29d371f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/TweetSent/sentiment.py
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import math,sys
+import pickle;
+import sklearn;
+import os;
+class TweetSent(object):
+
+ def __init__(self):
+
+ pickle_path = os.path.join(os.path.dirname(__file__), 'sentiment_pipeline3')
+ f = open(pickle_path,'rb')
+ self.pipeline = pickle.load(f)
+ f.close()
+
+ def sentiment(self, *args):
+ return self.pipeline.predict(args[0])[0].item()
diff --git a/asterixdb/asterix-app/src/test/resources/TweetSent/sentiment_pipeline3 b/asterixdb/asterix-app/src/test/resources/TweetSent/sentiment_pipeline3
new file mode 100644
index 0000000..4a1eba0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/TweetSent/sentiment_pipeline3
Binary files differ
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.1.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.1.lib.sqlpp
index 592653c..3dc6eb6 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.1.lib.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.1.lib.sqlpp
@@ -16,4 +16,4 @@
* specific language governing permissions and limitations
* under the License.
*/
-install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip
\ No newline at end of file
+install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.ddl.sqlpp
index cb57494..65733e4 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.ddl.sqlpp
@@ -16,4 +16,4 @@
* specific language governing permissions and limitations
* under the License.
*/
-DROP DATAVERSE externallibtest;
\ No newline at end of file
+DROP DATAVERSE externallibtest;
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.0.ddl.sqlpp
similarity index 81%
copy from asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.0.ddl.sqlpp
index d4788f8..76cc70d 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.0.ddl.sqlpp
@@ -16,11 +16,5 @@
* specific language governing permissions and limitations
* under the License.
*/
-
-package org.apache.asterix.om.functions;
-
-// WARNING: These values are stored in function metadata. Do not rename.
-public enum ExternalFunctionLanguage {
- JAVA,
- PYTHON
-}
\ No newline at end of file
+DROP DATAVERSE externallibtest if exists;
+CREATE DATAVERSE externallibtest;
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp
similarity index 81%
copy from asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp
index d4788f8..1650910 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.1.lib.sqlpp
@@ -16,11 +16,4 @@
* specific language governing permissions and limitations
* under the License.
*/
-
-package org.apache.asterix.om.functions;
-
-// WARNING: These values are stored in function metadata. Do not rename.
-public enum ExternalFunctionLanguage {
- JAVA,
- PYTHON
-}
\ No newline at end of file
+install externallibtest testlib admin admin target/TweetSent.pyz
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.2.ddl.sqlpp
similarity index 81%
copy from asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.2.ddl.sqlpp
index d4788f8..ff39aee 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.2.ddl.sqlpp
@@ -16,11 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
+ USE externallibtest;
-package org.apache.asterix.om.functions;
-
-// WARNING: These values are stored in function metadata. Do not rename.
-public enum ExternalFunctionLanguage {
- JAVA,
- PYTHON
-}
\ No newline at end of file
+create function sentiment(s) language python as "testlib","sentiment:TweetSent";
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.3.query.sqlpp
similarity index 81%
copy from asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.3.query.sqlpp
index d4788f8..6a3797c 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.3.query.sqlpp
@@ -16,11 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+use externallibtest;
-package org.apache.asterix.om.functions;
+sentiment("bad");
-// WARNING: These values are stored in function metadata. Do not rename.
-public enum ExternalFunctionLanguage {
- JAVA,
- PYTHON
-}
\ No newline at end of file
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.4.query.sqlpp
similarity index 81%
copy from asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.4.query.sqlpp
index d4788f8..b40e26b 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.4.query.sqlpp
@@ -16,11 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+use externallibtest;
-package org.apache.asterix.om.functions;
+sentiment("great");
-// WARNING: These values are stored in function metadata. Do not rename.
-public enum ExternalFunctionLanguage {
- JAVA,
- PYTHON
-}
\ No newline at end of file
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.5.query.sqlpp
similarity index 81%
rename from asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.5.query.sqlpp
index d4788f8..b2d69bf 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.5.query.sqlpp
@@ -16,11 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+use externallibtest;
-package org.apache.asterix.om.functions;
+sentiment("meh");
-// WARNING: These values are stored in function metadata. Do not rename.
-public enum ExternalFunctionLanguage {
- JAVA,
- PYTHON
-}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.6.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.6.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.6.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysentiment/mysentiment.6.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.6.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.7.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.6.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.7.ddl.sqlpp
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.0.ddl.sqlpp
similarity index 81%
copy from asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.0.ddl.sqlpp
index d4788f8..76cc70d 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.0.ddl.sqlpp
@@ -16,11 +16,5 @@
* specific language governing permissions and limitations
* under the License.
*/
-
-package org.apache.asterix.om.functions;
-
-// WARNING: These values are stored in function metadata. Do not rename.
-public enum ExternalFunctionLanguage {
- JAVA,
- PYTHON
-}
\ No newline at end of file
+DROP DATAVERSE externallibtest if exists;
+CREATE DATAVERSE externallibtest;
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.1.lib.sqlpp
similarity index 81%
copy from asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.1.lib.sqlpp
index d4788f8..1650910 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.1.lib.sqlpp
@@ -16,11 +16,4 @@
* specific language governing permissions and limitations
* under the License.
*/
-
-package org.apache.asterix.om.functions;
-
-// WARNING: These values are stored in function metadata. Do not rename.
-public enum ExternalFunctionLanguage {
- JAVA,
- PYTHON
-}
\ No newline at end of file
+install externallibtest testlib admin admin target/TweetSent.pyz
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.2.ddl.sqlpp
similarity index 81%
copy from asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.2.ddl.sqlpp
index d4788f8..b1c5b10 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.2.ddl.sqlpp
@@ -16,11 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
+ USE externallibtest;
-package org.apache.asterix.om.functions;
-
-// WARNING: These values are stored in function metadata. Do not rename.
-public enum ExternalFunctionLanguage {
- JAVA,
- PYTHON
-}
\ No newline at end of file
+create function system(s: string) language python as "testlib","os";
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.3.query.sqlpp
similarity index 81%
copy from asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.3.query.sqlpp
index d4788f8..fadd151 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.3.query.sqlpp
@@ -17,10 +17,11 @@
* under the License.
*/
-package org.apache.asterix.om.functions;
+/*
+ * Negative test case, should fail. Tries to call python builtins directly.
+ */
-// WARNING: These values are stored in function metadata. Do not rename.
-public enum ExternalFunctionLanguage {
- JAVA,
- PYTHON
-}
\ No newline at end of file
+use externallibtest;
+
+system("/bin/true");
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.6.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.4.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.6.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python-fn-escape/python-fn-escape.4.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.2.ddl.sqlpp
index f91005a..f87c3d7 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.2.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.2.ddl.sqlpp
@@ -18,5 +18,5 @@
*/
use externallibtest;
-create function typeValidation(a: int32, b: float, c:string, d:double, e:boolean, f:point, g:date, h:datetime, k:line, l:circle, m:rectangle) returns string language java as "testlib","org.apache.asterix.external.library.TypeValidationFunctionFactory";
+create function typeValidation(a: int32, b: float, c:string, d:double, e:boolean, f:date, g: datetime) returns string language java as "testlib","org.apache.asterix.external.library.TypeValidationFunctionFactory";
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.3.query.sqlpp
index 48ae8f6..358b28a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.3.query.sqlpp
@@ -17,4 +17,5 @@
* under the License.
*/
use externallibtest;
-typeValidation(907, 9.07, "907", 9.07, true, create_point(1.0, 1.0),date("2013-01-01"), datetime("1989-09-07T12:13:14.039Z"), create_line(create_point(1.0, 1.0), create_point(2.0, 2.0)), create_circle(create_point(1.0, 1.0), 2.0), create_rectangle(create_point(1.0, 1.0), create_point(2.0, 2.0)));
+typeValidation(907, 9.07, "907", 9.07, true, date("2013-01-01"), datetime("1989-09-07T12:13:14.039Z"));
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 50a901f..f68a897 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -22,7 +22,7 @@
"log\.dir" : "logs/",
"log\.level" : "INFO",
"max\.wait\.active\.cluster" : 60,
- "max.web.request.size" : 52428800,
+ "max.web.request.size" : 209715200,
"messaging\.frame\.count" : 512,
"messaging\.frame\.size" : 4096,
"metadata\.callback\.port" : 0,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 318206d..47c889b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -22,7 +22,7 @@
"log\.dir" : "logs/",
"log\.level" : "WARN",
"max\.wait\.active\.cluster" : 60,
- "max.web.request.size" : 52428800,
+ "max.web.request.size" : 209715200,
"messaging\.frame\.count" : 512,
"messaging\.frame\.size" : 4096,
"metadata\.callback\.port" : 0,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 69e9b0e..eb5be43 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -22,7 +22,7 @@
"log\.dir" : "logs/",
"log\.level" : "WARN",
"max\.wait\.active\.cluster" : 60,
- "max.web.request.size" : 52428800,
+ "max.web.request.size" : 209715200,
"messaging\.frame\.count" : 512,
"messaging\.frame\.size" : 4096,
"metadata\.callback\.port" : 0,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysentiment/mysentiment.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysentiment/mysentiment.1.adm
new file mode 100644
index 0000000..573541a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysentiment/mysentiment.1.adm
@@ -0,0 +1 @@
+0
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysentiment/mysentiment.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysentiment/mysentiment.2.adm
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysentiment/mysentiment.2.adm
@@ -0,0 +1 @@
+1
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysentiment/mysentiment.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysentiment/mysentiment.3.adm
new file mode 100644
index 0000000..573541a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysentiment/mysentiment.3.adm
@@ -0,0 +1 @@
+0
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/type_validation/type_validation.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/type_validation/type_validation.1.adm
index 874b506..8f4dc82 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/type_validation/type_validation.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/type_validation/type_validation.1.adm
@@ -1 +1 @@
-"907 9.07 \"907\" 9.07 TRUE point: { x: 1.0, y: 1.0 } \"date\": { 2013-01-01 } datetime: { 1989-09-07T12:13:14.039Z } line: { p1: point: { x: 1.0, y: 1.0 }, p2: point: { x: 2.0, y: 2.0 }} circle: { \"center\": point: { x: 1.0, y: 1.0 }, \"radius\":2.0} rectangle: { p1: point: { x: 1.0, y: 1.0 }, p2: point: { x: 2.0, y: 2.0 }}"
\ No newline at end of file
+"907 9.07 \"907\" 9.07 TRUE \"date\": { 2013-01-01 } datetime: { 1989-09-07T12:13:14.039Z } "
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-function/feed-with-external-function.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-function/feed-with-external-function.1.adm
index 0f7eb82..e69de29 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-function/feed-with-external-function.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-function/feed-with-external-function.1.adm
@@ -1,21 +0,0 @@
-{ "id": 21, "tweetid": 69902639026020352, "loc": point("34.5,-100.5"), "time": datetime("2011-05-15T16:11:02.000Z"), "text": "thats that smokers cough maam <<<<<--- @x_incredibleL :: Allergies. i got that "cough" lol", "mentionedUsers": [ "@x_incredibleL" ] }
-{ "id": 22, "tweetid": 69988755800465408, "loc": point("34.5,-97.5"), "time": datetime("2011-05-15T21:53:14.000Z"), "text": "Allergies fuckin over me..#damn", "mentionedUsers": [ ] }
-{ "id": 23, "tweetid": 69940039605432320, "loc": point("34.5,-97.5"), "time": datetime("2011-05-15T18:39:39.000Z"), "text": "Natural Asthma Remedy - Deal With Your Asthma in a Natural Way.. Allergies", "mentionedUsers": [ ] }
-{ "id": 24, "tweetid": 69834276929159169, "loc": point("25.5,-100.5"), "time": datetime("2011-05-15T11:39:23.000Z"), "text": "Damn Allergies... sneezing like crazy! >_<", "mentionedUsers": [ ] }
-{ "id": 25, "tweetid": 69950146787553281, "loc": point("25.5,-97.5"), "time": datetime("2011-05-15T19:19:49.000Z"), "text": "pass me an asthma pump", "mentionedUsers": [ ] }
-{ "id": 26, "tweetid": 69754524767756289, "loc": point("25.5,-97.5"), "time": datetime("2011-05-15T06:22:29.000Z"), "text": "Never knew allergies could actually keep me from sleeping", "mentionedUsers": [ ] }
-{ "id": 27, "tweetid": 69999864498487297, "loc": point("25.5,-80.5"), "time": datetime("2011-05-15T22:37:22.000Z"), "text": "@ItsCrystal320 gooodd mommy! Except my allergies have been acting up :( and Im having issues with you know who. Smh nothing new. Lol", "mentionedUsers": [ "@ItsCrystal320" ] }
-{ "id": 28, "tweetid": 69996796616777728, "loc": point("25.5,-80.5"), "time": datetime("2011-05-15T22:25:11.000Z"), "text": "My allergies act up so much while Im in this house!!! Idk why! Sneezing, now my eye is swollen!! Smh.", "mentionedUsers": [ ] }
-{ "id": 29, "tweetid": 69977295351316480, "loc": point("25.5,-80.5"), "time": datetime("2011-05-15T21:07:41.000Z"), "text": "@GOLDenNote6 lmmmaaaoooo!!!! nnnnooo! ur the one that needs the asthma pump!", "mentionedUsers": [ "@GOLDenNote6" ] }
-{ "id": 30, "tweetid": 69972022586912768, "loc": point("25.5,-80.5"), "time": datetime("2011-05-15T20:46:44.000Z"), "text": "@TinaLee90 hell yeah ! He snapped cause she got allergies and heavy she be snorting and coughing while he trying to study", "mentionedUsers": [ "@TinaLee90" ] }
-{ "id": 31, "tweetid": 69965044678524928, "loc": point("25.5,-80.5"), "time": datetime("2011-05-15T20:19:01.000Z"), "text": "Back home and my ears begin to itch!!! Omg allergies go away please! #thingsicanlivewithout", "mentionedUsers": [ ] }
-{ "id": 32, "tweetid": 69961997680246784, "loc": point("25.5,-80.5"), "time": datetime("2011-05-15T20:06:54.000Z"), "text": "@BravoAndy allergies acting up again or you just digging the glasses? Haha u rock it though!", "mentionedUsers": [ "@BravoAndy" ] }
-{ "id": 33, "tweetid": 69946356248215552, "loc": point("25.5,-80.5"), "time": datetime("2011-05-15T19:04:45.000Z"), "text": "My allergies act up at the worst times -_-", "mentionedUsers": [ ] }
-{ "id": 34, "tweetid": 69929466691993600, "loc": point("25.5,-80.5"), "time": datetime("2011-05-15T17:57:38.000Z"), "text": "Hate being sick!!! -_____- I hate you allergies! :/", "mentionedUsers": [ ] }
-{ "id": 35, "tweetid": 69928014615556096, "loc": point("25.5,-80.5"), "time": datetime("2011-05-15T17:51:52.000Z"), "text": "Allergies please go away :(", "mentionedUsers": [ ] }
-{ "id": 36, "tweetid": 69916338092654592, "loc": point("25.5,-80.5"), "time": datetime("2011-05-15T17:05:28.000Z"), "text": "I feel tired....i got asthma :( but it was still an awesome birthday", "mentionedUsers": [ ] }
-{ "id": 37, "tweetid": 69911241975529474, "loc": point("25.5,-80.5"), "time": datetime("2011-05-15T16:45:13.000Z"), "text": "Cant stand that asthma commercial with the gold fish -__-", "mentionedUsers": [ ] }
-{ "id": 38, "tweetid": 69910467233062912, "loc": point("25.5,-80.5"), "time": datetime("2011-05-15T16:42:08.000Z"), "text": "@PapisFavWave whats wrong? Got a cold? Asthma ?", "mentionedUsers": [ "@PapisFavWave" ] }
-{ "id": 39, "tweetid": 69908652202536961, "loc": point("25.5,-80.5"), "time": datetime("2011-05-15T16:34:56.000Z"), "text": "My allergies are killing me!", "mentionedUsers": [ ] }
-{ "id": 40, "tweetid": 69897794273546240, "loc": point("25.5,-80.5"), "time": datetime("2011-05-15T15:51:47.000Z"), "text": "and allergies", "mentionedUsers": [ ] }
-{ "id": 41, "tweetid": 69893733449080832, "loc": point("25.5,-80.5"), "time": datetime("2011-05-15T15:35:39.000Z"), "text": "Repeated splashing of water about the skin, specifically following an exposure to pollution and dirt, makes sure... http://bit.ly/mnWnJo", "mentionedUsers": [ ] }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
new file mode 100644
index 0000000..6d03162
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
@@ -0,0 +1,39 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ ~
+ -->
+<test-suite
+ xmlns="urn:xml.testframework.asterix.apache.org"
+ ResultOffsetPath="results"
+ QueryOffsetPath="queries_sqlpp"
+ QueryFileExtension=".sqlpp">
+
+ <test-group name="external-library-python">
+ <test-case FilePath="external-library">
+ <compilation-unit name="mysentiment">
+ <output-dir compare="Text">mysentiment</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-library">
+ <compilation-unit name="python-fn-escape">
+ <output-dir compare="Text">python-fn-escape</output-dir>
+ <expected-error>'NoneType' object is not callable</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+</test-suite>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
index 0acab5c..033b751 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
@@ -52,7 +52,7 @@
NC_JAVA_OPTS(STRING, "-Xmx1024m", "The JVM options passed to the node controller process(es) by managix"),
MAX_WEB_REQUEST_SIZE(
UNSIGNED_INTEGER,
- StorageUtil.getIntSizeInBytes(50, StorageUtil.StorageUnit.MEGABYTE),
+ StorageUtil.getIntSizeInBytes(200, StorageUtil.StorageUnit.MEGABYTE),
"The maximum accepted web request size in bytes"),
REQUESTS_ARCHIVE_SIZE(UNSIGNED_INTEGER, 50, "The maximum number of archived requests to maintain"),
CREDENTIAL_FILE(
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/functions/ExternalFunctionLanguage.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/functions/ExternalFunctionLanguage.java
new file mode 100644
index 0000000..8c4cf68
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/functions/ExternalFunctionLanguage.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.asterix.common.functions;
+
+// WARNING: These values are stored in function metadata. Do not rename.
+public enum ExternalFunctionLanguage {
+ JAVA,
+ PYTHON
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibrary.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibrary.java
new file mode 100644
index 0000000..164c215
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibrary.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.asterix.common.library;
+
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+
+public interface ILibrary {
+
+ ExternalFunctionLanguage getLanguage();
+
+ void close();
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java
index 870a6dc..fdaaaeb 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java
@@ -19,45 +19,20 @@
package org.apache.asterix.common.library;
-import java.net.URLClassLoader;
-import java.util.List;
+import java.io.File;
+import java.io.IOException;
+import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.metadata.DataverseName;
-import org.apache.hyracks.algebricks.common.utils.Pair;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
public interface ILibraryManager {
- /**
- * Registers the library class loader with the external library manager.
- * <code>dataverseName</code> and <code>libraryName</code> uniquely identifies a class loader.
- *
- * @param dataverseName
- * @param libraryName
- * @param classLoader
- */
- void registerLibraryClassLoader(DataverseName dataverseName, String libraryName, URLClassLoader classLoader)
- throws HyracksDataException;
+ void setUpDeployedLibrary(String path) throws IOException, AsterixException;
- /**
- * @return all registered libraries.
- */
- List<Pair<DataverseName, String>> getAllLibraries();
+ void scanLibraries(File appDir);
- /**
- * De-registers a library class loader.
- *
- * @param dataverseName
- * @param libraryName
- */
- void deregisterLibraryClassLoader(DataverseName dataverseName, String libraryName);
+ void deregister(DataverseName dv, String name);
- /**
- * Finds a class loader for a given pair of dataverse name and library name.
- *
- * @param dataverseName
- * @param libraryName
- * @return the library class loader associated with the dataverse and library.
- */
- ClassLoader getLibraryClassLoader(DataverseName dataverseName, String libraryName);
+ ILibrary getLibrary(DataverseName dvName, String libraryName);
+
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/LibraryDescriptor.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/LibraryDescriptor.java
new file mode 100644
index 0000000..9dd2a3d
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/LibraryDescriptor.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.asterix.common.library;
+
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+/**
+ * The information needed to libraries at startup
+ */
+public class LibraryDescriptor implements IJsonSerializable {
+
+ private static final long serialVersionUID = 1L;
+ public static final String DESCRIPTOR_NAME = "descriptor.json";
+ private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
+ /**
+ * The library's language
+ */
+ private final ExternalFunctionLanguage lang;
+
+ public LibraryDescriptor(ExternalFunctionLanguage lang) {
+ this.lang = lang;
+ }
+
+ public ExternalFunctionLanguage getLang() {
+ return lang;
+ }
+
+ public JsonNode toJson(IPersistedResourceRegistry registry) {
+ ObjectNode jsonNode = registry.getClassIdentifier(LibraryDescriptor.class, serialVersionUID);
+ jsonNode.put("lang", lang.name());
+ return jsonNode;
+ }
+
+ public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json) {
+ final ExternalFunctionLanguage lang = ExternalFunctionLanguage.valueOf(json.get("lang").asText());
+ return new LibraryDescriptor(lang);
+ }
+}
diff --git a/asterixdb/asterix-external-data/pom.xml b/asterixdb/asterix-external-data/pom.xml
index 3f1c6d4..08ffb16 100644
--- a/asterixdb/asterix-external-data/pom.xml
+++ b/asterixdb/asterix-external-data/pom.xml
@@ -101,6 +101,7 @@
<includes>
<include>**/*.class</include>
<include>**/*.txt</include>
+ <include>**/*.py</include>
<include>**/NOTICE</include>
<include>**/LICENSE</include>
<include>**/*.properties</include>
@@ -205,6 +206,7 @@
<usedDependency>com.sun.xml.bind:jaxb-core</usedDependency>
<usedDependency>com.sun.xml.bind:jaxb-impl</usedDependency>
<usedDependency>com.sun.activation:javax.activation</usedDependency>
+ <usedDependency>net.razorvine:serpent</usedDependency>
</usedDependencies>
</configuration>
</plugin>
@@ -305,11 +307,19 @@
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-dataflow-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-control-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-am-lsm-btree</artifactId>
</dependency>
<dependency>
@@ -434,5 +444,17 @@
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>net.razorvine</groupId>
+ <artifactId>pyrolite</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>net.razorvine</groupId>
+ <artifactId>serpent</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IFunctionHelper.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IFunctionHelper.java
index 91bce9c..15435ee 100755
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IFunctionHelper.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IFunctionHelper.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.api;
+import java.util.List;
import java.util.Map;
import org.apache.asterix.external.library.java.JTypeTag;
@@ -41,4 +42,6 @@
void reset();
Map<String, String> getParameters();
+
+ List<String> getExternalIdentifier();
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJObject.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJObject.java
index 9ea400e..79674ed 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJObject.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJObject.java
@@ -24,13 +24,18 @@
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public interface IJObject {
+public interface IJObject<T> {
IAType getIAType();
IAObject getIAObject();
+ void setValueGeneric(T o) throws HyracksDataException;
+
+ T getValueGeneric();
+
void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException;
void reset() throws HyracksDataException;
+
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryClassLoader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryClassLoader.java
similarity index 97%
rename from asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryClassLoader.java
rename to asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryClassLoader.java
index 9cf7584..1eac02a 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryClassLoader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryClassLoader.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.app.external;
+package org.apache.asterix.external.library;
import java.net.URL;
import java.net.URLClassLoader;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
index bc03de0..9e1461e 100755
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
@@ -18,69 +18,116 @@
*/
package org.apache.asterix.external.library;
+import static com.fasterxml.jackson.databind.MapperFeature.SORT_PROPERTIES_ALPHABETICALLY;
+import static com.fasterxml.jackson.databind.SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS;
+import static org.apache.asterix.common.library.LibraryDescriptor.DESCRIPTOR_NAME;
+
+import java.io.File;
import java.io.IOException;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
+import java.nio.file.Files;
+import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+import org.apache.asterix.common.library.ILibrary;
import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.common.library.LibraryDescriptor;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+
public class ExternalLibraryManager implements ILibraryManager {
- private final Map<Pair<DataverseName, String>, URLClassLoader> libraryClassLoaders = new HashMap<>();
+ protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ static {
+ OBJECT_MAPPER.enable(SerializationFeature.INDENT_OUTPUT);
+ OBJECT_MAPPER.configure(SORT_PROPERTIES_ALPHABETICALLY, true);
+ OBJECT_MAPPER.configure(ORDER_MAP_ENTRIES_BY_KEYS, true);
+ }
+
private static final Logger LOGGER = LogManager.getLogger();
+ private final Map<Pair<DataverseName, String>, ILibrary> libraries = Collections.synchronizedMap(new HashMap());
+ private final IPersistedResourceRegistry reg;
- @Override
- public void registerLibraryClassLoader(DataverseName dataverseName, String libraryName,
- URLClassLoader classLoader) {
- Pair<DataverseName, String> key = getKey(dataverseName, libraryName);
- synchronized (libraryClassLoaders) {
- if (libraryClassLoaders.get(key) != null) {
- return;
- }
- libraryClassLoaders.put(key, classLoader);
- }
+ public ExternalLibraryManager(File appDir, IPersistedResourceRegistry reg) {
+ this.reg = reg;
+ scanLibraries(appDir);
}
@Override
- public List<Pair<DataverseName, String>> getAllLibraries() {
- ArrayList<Pair<DataverseName, String>> libs = new ArrayList<>();
- synchronized (libraryClassLoaders) {
- libraryClassLoaders.forEach((key, value) -> libs.add(key));
- }
- return libs;
- }
-
- @Override
- public void deregisterLibraryClassLoader(DataverseName dataverseName, String libraryName) {
- Pair<DataverseName, String> key = getKey(dataverseName, libraryName);
- synchronized (libraryClassLoaders) {
- URLClassLoader cl = libraryClassLoaders.get(key);
- if (cl != null) {
+ public void scanLibraries(File appDir) {
+ File[] libs = appDir.listFiles((dir, name) -> dir.isDirectory());
+ if (libs != null) {
+ for (File lib : libs) {
+ String libraryPath = lib.getAbsolutePath();
try {
- cl.close();
- } catch (IOException e) {
- LOGGER.error("Unable to close UDF classloader!", e);
+ setUpDeployedLibrary(libraryPath);
+ } catch (AsterixException | IOException e) {
+ LOGGER.error("Unable to properly initialize external libraries", e);
}
- libraryClassLoaders.remove(key);
}
}
}
- @Override
- public ClassLoader getLibraryClassLoader(DataverseName dataverseName, String libraryName) {
+ public void register(DataverseName dataverseName, String libraryName, ILibrary library) {
Pair<DataverseName, String> key = getKey(dataverseName, libraryName);
- return libraryClassLoaders.get(key);
+ libraries.put(key, library);
+ }
+
+ @Override
+ public void deregister(DataverseName dataverseName, String libraryName) {
+ Pair<DataverseName, String> key = getKey(dataverseName, libraryName);
+ ILibrary cl = libraries.get(key);
+ if (cl != null) {
+ cl.close();
+ libraries.remove(key);
+ }
+ }
+
+ public void setUpDeployedLibrary(String libraryPath) throws IOException, AsterixException {
+ // get the installed library dirs
+ String[] parts = libraryPath.split(File.separator);
+ DataverseName catenatedDv = DataverseName.createFromCanonicalForm(parts[parts.length - 1]);
+ String name = catenatedDv.getParts().get(catenatedDv.getParts().size() - 1);
+ DataverseName dataverse = DataverseName.create(catenatedDv.getParts(), 0, catenatedDv.getParts().size() - 1);
+ try {
+ File langFile = new File(libraryPath, DESCRIPTOR_NAME);
+ final JsonNode jsonNode = OBJECT_MAPPER.readValue(Files.readAllBytes(langFile.toPath()), JsonNode.class);
+ LibraryDescriptor desc = (LibraryDescriptor) reg.deserialize(jsonNode);
+ ExternalFunctionLanguage libLang = desc.getLang();
+ switch (libLang) {
+ case JAVA:
+ register(dataverse, name, new JavaLibrary(libraryPath));
+ break;
+ case PYTHON:
+ register(dataverse, name, new PythonLibrary(libraryPath));
+ break;
+ default:
+ throw new IllegalStateException("Library path file refers to unknown language");
+ }
+ } catch (IOException | AsterixException e) {
+ LOGGER.error("Failed to initialized library", e);
+ throw e;
+ }
+ }
+
+ @Override
+ public ILibrary getLibrary(DataverseName dataverseName, String libraryName) {
+ Pair<DataverseName, String> key = getKey(dataverseName, libraryName);
+ return libraries.get(key);
}
private static Pair<DataverseName, String> getKey(DataverseName dataverseName, String libraryName) {
- return new Pair(dataverseName, libraryName);
+ return new Pair<>(dataverseName, libraryName);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
index e406f9f..4a073c9 100755
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
@@ -29,7 +29,6 @@
import org.apache.asterix.external.api.IFunctionFactory;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.types.IAType;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -40,20 +39,21 @@
class ExternalScalarJavaFunctionEvaluator extends ExternalScalarFunctionEvaluator {
- protected final IExternalScalarFunction externalFunctionInstance;
- protected final IPointable inputVal = VoidPointable.FACTORY.createPointable();
- protected final ArrayBackedValueStorage resultBuffer = new ArrayBackedValueStorage();
+ private final IExternalScalarFunction externalFunctionInstance;
+ private final IPointable inputVal = VoidPointable.FACTORY.createPointable();
+ private final ArrayBackedValueStorage resultBuffer = new ArrayBackedValueStorage();
protected final JavaFunctionHelper functionHelper;
- public ExternalScalarJavaFunctionEvaluator(IExternalFunctionInfo finfo, IScalarEvaluatorFactory[] args,
- IAType[] argTypes, IEvaluatorContext context) throws HyracksDataException {
+ ExternalScalarJavaFunctionEvaluator(IExternalFunctionInfo finfo, IScalarEvaluatorFactory[] args, IAType[] argTypes,
+ IEvaluatorContext context) throws HyracksDataException {
super(finfo, args, argTypes, context);
DataverseName functionDataverse = FunctionSignature.getDataverseName(finfo.getFunctionIdentifier());
String functionLibrary = finfo.getLibrary();
functionHelper = new JavaFunctionHelper(finfo, argTypes, resultBuffer);
- ClassLoader libraryClassLoader = libraryManager.getLibraryClassLoader(functionDataverse, functionLibrary);
+ JavaLibrary lib = (JavaLibrary) libraryManager.getLibrary(functionDataverse, functionLibrary);
+ ClassLoader libraryClassLoader = lib.getClassLoader();
String classname = finfo.getExternalIdentifier().get(0).trim();
try {
Class<?> clazz = Class.forName(classname, true, libraryClassLoader);
@@ -86,7 +86,7 @@
}
}
- public void setArguments(IFrameTupleReference tuple) throws AlgebricksException, IOException {
+ public void setArguments(IFrameTupleReference tuple) throws IOException {
for (int i = 0; i < argEvals.length; i++) {
argEvals[i].evaluate(tuple, inputVal);
functionHelper.setArgument(i, inputVal);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
index dc06a72..e49c97e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
@@ -19,70 +19,194 @@
package org.apache.asterix.external.library;
+import java.io.DataOutput;
+import java.io.File;
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.ServerSocket;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.Objects;
import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.external.api.IJObject;
+import org.apache.asterix.external.library.java.JObjectPointableVisitor;
+import org.apache.asterix.external.library.java.base.JComplexObject;
+import org.apache.asterix.external.library.java.base.JObject;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
+import org.apache.asterix.om.pointables.AFlatValuePointable;
+import org.apache.asterix.om.pointables.AListVisitablePointable;
+import org.apache.asterix.om.pointables.ARecordVisitablePointable;
+import org.apache.asterix.om.pointables.PointableAllocator;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.EnumDeserializer;
import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.runtime.evaluators.functions.PointableHelper;
+import org.apache.asterix.om.types.TypeTagUtil;
+import org.apache.asterix.om.util.container.IObjectPool;
+import org.apache.asterix.om.util.container.ListObjectPool;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.resources.IDeallocatable;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.TaggedValuePointable;
import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
import org.apache.hyracks.dataflow.std.base.AbstractStateObject;
+import net.razorvine.pyro.PyroProxy;
+
class ExternalScalarPythonFunctionEvaluator extends ExternalScalarFunctionEvaluator {
private final PythonLibraryEvaluator libraryEvaluator;
+ private final ArrayBackedValueStorage resultBuffer = new ArrayBackedValueStorage();
+ private final PointableAllocator pointableAllocator;
+ private final JObjectPointableVisitor pointableVisitor;
+ private final Object[] argHolder;
+ private final IObjectPool<IJObject, IAType> reflectingPool = new ListObjectPool<>(JTypeObjectFactory.INSTANCE);
+ private final Map<IAType, TypeInfo> infoPool = new HashMap<>();
+ private static final String ENTRYPOINT = "entrypoint.py";
+ private static final String PY_NO_SITE_PKGS_OPT = "-S";
+ private static final String PY_NO_USER_PKGS_OPT = "-s";
+
private final IPointable[] argValues;
- public ExternalScalarPythonFunctionEvaluator(IExternalFunctionInfo finfo, IScalarEvaluatorFactory[] args,
+ ExternalScalarPythonFunctionEvaluator(IExternalFunctionInfo finfo, IScalarEvaluatorFactory[] args,
IAType[] argTypes, IEvaluatorContext ctx) throws HyracksDataException {
super(finfo, args, argTypes, ctx);
+
+ File pythonPath = new File(ctx.getServiceContext().getAppConfig().getString(NCConfig.Option.PYTHON_HOME));
+ this.pointableAllocator = new PointableAllocator();
+ this.pointableVisitor = new JObjectPointableVisitor();
+
DataverseName dataverseName = FunctionSignature.getDataverseName(finfo.getFunctionIdentifier());
- libraryEvaluator = PythonLibraryEvaluator.getInstance(dataverseName, finfo.getLibrary(), ctx.getTaskContext());
+ try {
+ libraryEvaluator = PythonLibraryEvaluator.getInstance(dataverseName, finfo, libraryManager, pythonPath,
+ ctx.getTaskContext());
+ } catch (IOException | InterruptedException e) {
+ throw new HyracksDataException("Failed to initialize Python", e);
+ }
argValues = new IPointable[args.length];
for (int i = 0; i < argValues.length; i++) {
argValues[i] = VoidPointable.FACTORY.createPointable();
}
+ this.argHolder = new Object[args.length];
}
@Override
public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
for (int i = 0, ln = argEvals.length; i < ln; i++) {
argEvals[i].evaluate(tuple, argValues[i]);
+ try {
+ setArgument(i, argValues[i]);
+ } catch (IOException e) {
+ throw new HyracksDataException("Error evaluating Python UDF", e);
+ }
}
- PointableHelper.setNull(result);
+ try {
+ Object res = libraryEvaluator.callPython(argHolder);
+ resultBuffer.reset();
+ wrap(res, resultBuffer.getDataOutput());
+ } catch (IOException e) {
+ throw new HyracksDataException("Error evaluating Python UDF", e);
+ }
+ result.set(resultBuffer.getByteArray(), resultBuffer.getStartOffset(), resultBuffer.getLength());
}
private static class PythonLibraryEvaluator extends AbstractStateObject implements IDeallocatable {
+ Process p;
+ PyroProxy remoteObj;
+ IExternalFunctionInfo finfo;
+ ILibraryManager libMgr;
+ File pythonHome;
- private PythonLibraryEvaluator(JobId jobId, PythonLibraryEvaluatorId evaluatorId) {
+ private PythonLibraryEvaluator(JobId jobId, PythonLibraryEvaluatorId evaluatorId, IExternalFunctionInfo finfo,
+ ILibraryManager libMgr, File pythonHome) {
super(jobId, evaluatorId);
+ this.finfo = finfo;
+ this.libMgr = libMgr;
+ this.pythonHome = pythonHome;
+
+ }
+
+ public void initialize() throws IOException, InterruptedException {
+ PythonLibraryEvaluatorId fnId = (PythonLibraryEvaluatorId) id;
+ List<String> externalIdents = finfo.getExternalIdentifier();
+ PythonLibrary library = (PythonLibrary) libMgr.getLibrary(fnId.dataverseName, fnId.libraryName);
+ String wd = library.getFile().getAbsolutePath();
+ int port = getFreeHighPort();
+ String packageModule = externalIdents.get(0);
+ String clazz = "None";
+ String fn;
+ if (externalIdents.size() > 2) {
+ clazz = externalIdents.get(1);
+ fn = externalIdents.get(2);
+ } else {
+ fn = externalIdents.get(1);
+ }
+ ProcessBuilder pb = new ProcessBuilder(pythonHome.getAbsolutePath(), PY_NO_SITE_PKGS_OPT,
+ PY_NO_USER_PKGS_OPT, ENTRYPOINT, Integer.toString(port), packageModule, clazz, fn);
+ pb.directory(new File(wd));
+ pb.environment().clear();
+ pb.inheritIO();
+ p = pb.start();
+ remoteObj = new PyroProxy("127.0.0.1", port, "nextTuple");
+ waitForPython();
+ }
+
+ Object callPython(Object[] arguments) throws IOException {
+ return remoteObj.call("nextTuple", arguments);
}
@Override
public void deallocate() {
+ p.destroyForcibly();
}
- private static PythonLibraryEvaluator getInstance(DataverseName dataverseName, String libraryName,
- IHyracksTaskContext ctx) {
- PythonLibraryEvaluatorId evaluatorId = new PythonLibraryEvaluatorId(dataverseName, libraryName);
+ private static PythonLibraryEvaluator getInstance(DataverseName dataverseName, IExternalFunctionInfo finfo,
+ ILibraryManager libMgr, File pythonHome, IHyracksTaskContext ctx)
+ throws IOException, InterruptedException {
+ PythonLibraryEvaluatorId evaluatorId = new PythonLibraryEvaluatorId(dataverseName, finfo.getLibrary());
PythonLibraryEvaluator evaluator = (PythonLibraryEvaluator) ctx.getStateObject(evaluatorId);
if (evaluator == null) {
- evaluator = new PythonLibraryEvaluator(ctx.getJobletContext().getJobId(), evaluatorId);
+ evaluator = new PythonLibraryEvaluator(ctx.getJobletContext().getJobId(), evaluatorId, finfo, libMgr,
+ pythonHome);
+ evaluator.initialize();
ctx.registerDeallocatable(evaluator);
ctx.setStateObject(evaluator);
}
return evaluator;
}
+
+ private int getFreeHighPort() throws IOException {
+ int port;
+ try (ServerSocket socket = new ServerSocket(0)) {
+ socket.setReuseAddress(true);
+ port = socket.getLocalPort();
+ }
+ return port;
+ }
+
+ private void waitForPython() throws IOException, InterruptedException {
+ for (int i = 0; i < 100; i++) {
+ try {
+ remoteObj.call("ping");
+ break;
+ } catch (ConnectException e) {
+ Thread.sleep(100);
+ }
+ }
+ }
}
private static final class PythonLibraryEvaluatorId {
@@ -111,4 +235,78 @@
return Objects.hash(dataverseName, libraryName);
}
}
-}
\ No newline at end of file
+
+ private void setArgument(int index, IValueReference valueReference) throws IOException {
+ IVisitablePointable pointable;
+ IJObject jobj;
+ IAType type = argTypes[index];
+ TypeInfo info;
+ switch (type.getTypeTag()) {
+ case OBJECT:
+ pointable = pointableAllocator.allocateRecordValue(type);
+ pointable.set(valueReference);
+ info = getTypeInfo(type);
+ jobj = pointableVisitor.visit((ARecordVisitablePointable) pointable, info);
+ break;
+ case ARRAY:
+ case MULTISET:
+ pointable = pointableAllocator.allocateListValue(type);
+ pointable.set(valueReference);
+ info = getTypeInfo(type);
+ jobj = pointableVisitor.visit((AListVisitablePointable) pointable, info);
+ break;
+ case ANY:
+ TaggedValuePointable pointy = TaggedValuePointable.FACTORY.createPointable();
+ pointy.set(valueReference);
+ ATypeTag rtTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(pointy.getTag());
+ IAType rtType = TypeTagUtil.getBuiltinTypeByTag(rtTypeTag);
+ info = getTypeInfo(rtType);
+ switch (rtTypeTag) {
+ case OBJECT:
+ pointable = pointableAllocator.allocateRecordValue(rtType);
+ pointable.set(valueReference);
+ jobj = pointableVisitor.visit((ARecordVisitablePointable) pointable, info);
+ break;
+ case ARRAY:
+ case MULTISET:
+ pointable = pointableAllocator.allocateListValue(rtType);
+ pointable.set(valueReference);
+ jobj = pointableVisitor.visit((AListVisitablePointable) pointable, info);
+ break;
+ default:
+ pointable = pointableAllocator.allocateFieldValue(rtType);
+ pointable.set(valueReference);
+ jobj = pointableVisitor.visit((AFlatValuePointable) pointable, info);
+ break;
+ }
+ break;
+ default:
+ pointable = pointableAllocator.allocateFieldValue(type);
+ pointable.set(valueReference);
+ info = getTypeInfo(type);
+ jobj = pointableVisitor.visit((AFlatValuePointable) pointable, info);
+ break;
+ }
+ argHolder[index] = jobj.getValueGeneric();
+ }
+
+ private TypeInfo getTypeInfo(IAType type) {
+ TypeInfo typeInfo = infoPool.get(type);
+ if (typeInfo == null) {
+ typeInfo = new TypeInfo(reflectingPool, type, type.getTypeTag());
+ infoPool.put(type, typeInfo);
+ }
+ return typeInfo;
+ }
+
+ private void wrap(Object o, DataOutput out) throws HyracksDataException {
+ Class concrete = o.getClass();
+ IAType asxConv = JObject.convertType(concrete);
+ IJObject res = reflectingPool.allocate(asxConv);
+ if (res instanceof JComplexObject) {
+ ((JComplexObject) res).setPool(reflectingPool);
+ }
+ res.setValueGeneric(o);
+ res.serialize(out, true);
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/JTypeObjectFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/JTypeObjectFactory.java
index 415b13d..bf8fadf 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/JTypeObjectFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/JTypeObjectFactory.java
@@ -20,7 +20,6 @@
import org.apache.asterix.external.api.IJObject;
import org.apache.asterix.external.library.java.base.JBoolean;
-import org.apache.asterix.external.library.java.base.JCircle;
import org.apache.asterix.external.library.java.base.JDate;
import org.apache.asterix.external.library.java.base.JDateTime;
import org.apache.asterix.external.library.java.base.JDouble;
@@ -28,16 +27,11 @@
import org.apache.asterix.external.library.java.base.JFloat;
import org.apache.asterix.external.library.java.base.JInt;
import org.apache.asterix.external.library.java.base.JInterval;
-import org.apache.asterix.external.library.java.base.JLine;
import org.apache.asterix.external.library.java.base.JLong;
import org.apache.asterix.external.library.java.base.JMissing;
import org.apache.asterix.external.library.java.base.JNull;
import org.apache.asterix.external.library.java.base.JOrderedList;
-import org.apache.asterix.external.library.java.base.JPoint;
-import org.apache.asterix.external.library.java.base.JPoint3D;
-import org.apache.asterix.external.library.java.base.JPolygon;
import org.apache.asterix.external.library.java.base.JRecord;
-import org.apache.asterix.external.library.java.base.JRectangle;
import org.apache.asterix.external.library.java.base.JString;
import org.apache.asterix.external.library.java.base.JTime;
import org.apache.asterix.external.library.java.base.JUnorderedList;
@@ -74,24 +68,6 @@
case BOOLEAN:
retValue = new JBoolean(false);
break;
- case CIRCLE:
- retValue = new JCircle(new JPoint(0, 0), 0);
- break;
- case POINT:
- retValue = new JPoint(0, 0);
- break;
- case POINT3D:
- retValue = new JPoint3D(0, 0, 0);
- break;
- case POLYGON:
- retValue = new JPolygon(new JPoint[] {});
- break;
- case LINE:
- retValue = new JLine(new JPoint(0, 0), new JPoint(0, 0));
- break;
- case RECTANGLE:
- retValue = new JRectangle(new JPoint(0, 0), new JPoint(1, 1));
- break;
case DATE:
retValue = new JDate(0);
break;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/JavaFunctionHelper.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/JavaFunctionHelper.java
index 6e28f98..5f6e1eb 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/JavaFunctionHelper.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/JavaFunctionHelper.java
@@ -20,9 +20,9 @@
import java.io.IOException;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
-import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.external.api.IFunctionHelper;
@@ -62,7 +62,7 @@
private boolean isValidResult = false;
- public JavaFunctionHelper(IExternalFunctionInfo finfo, IAType[] argTypes, IDataOutputProvider outputProvider) {
+ JavaFunctionHelper(IExternalFunctionInfo finfo, IAType[] argTypes, IDataOutputProvider outputProvider) {
this.finfo = finfo;
this.outputProvider = outputProvider;
this.pointableVisitor = new JObjectPointableVisitor();
@@ -99,10 +99,7 @@
if (expectedType.equals(BuiltinType.ANY)) {
return false;
}
- if (!expectedType.deepEqual(result.getIAType())) {
- return true;
- }
- return false;
+ return !expectedType.deepEqual(result.getIAType());
}
/**
@@ -116,9 +113,9 @@
return this.isValidResult;
}
- public void setArgument(int index, IValueReference valueReference) throws IOException, AsterixException {
- IVisitablePointable pointable = null;
- IJObject jObject = null;
+ void setArgument(int index, IValueReference valueReference) throws IOException {
+ IVisitablePointable pointable;
+ IJObject jObject;
IAType type = argTypes[index];
switch (type.getTypeTag()) {
case OBJECT:
@@ -230,4 +227,10 @@
public Map<String, String> getParameters() {
return parameters;
}
+
+ @Override
+ public List<String> getExternalIdentifier() {
+ return finfo.getExternalIdentifier();
+ }
+
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/JavaLibrary.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/JavaLibrary.java
new file mode 100644
index 0000000..c7e7c09
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/JavaLibrary.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.asterix.external.library;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+import org.apache.asterix.common.library.ILibrary;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class JavaLibrary implements ILibrary {
+
+ private final ExternalLibraryClassLoader cl;
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ public JavaLibrary(String libraryPath) throws AsterixException, MalformedURLException {
+
+ File installDir = new File(libraryPath);
+
+ // get a reference to the specific library dir
+ File libDir = installDir;
+
+ FilenameFilter jarFileFilter = (dir, name) -> name.endsWith(".jar");
+
+ // Get the jar file <Allow only a single jar file>
+ String[] jarsInLibDir = libDir.list(jarFileFilter);
+ if (jarsInLibDir.length > 1) {
+ throw new AsterixException("Incorrect library structure: found multiple library jars");
+ }
+ if (jarsInLibDir.length <= 0) {
+ throw new AsterixException("Incorrect library structure: could not find library jar");
+ }
+
+ File libJar = new File(libDir, jarsInLibDir[0]);
+ // get the jar dependencies
+ File libDependencyDir = new File(libDir.getAbsolutePath() + File.separator + "lib");
+ int numDependencies = 1;
+ String[] libraryDependencies = null;
+ if (libDependencyDir.exists()) {
+ libraryDependencies = libDependencyDir.list(jarFileFilter);
+ numDependencies += libraryDependencies.length;
+ }
+
+ ClassLoader parentClassLoader = JavaLibrary.class.getClassLoader();
+ URL[] urls = new URL[numDependencies];
+ int count = 0;
+ // get url of library
+ urls[count++] = libJar.toURI().toURL();
+
+ // get urls for dependencies
+ if (libraryDependencies != null && libraryDependencies.length > 0) {
+ for (String dependency : libraryDependencies) {
+ File file = new File(libDependencyDir + File.separator + dependency);
+ urls[count++] = file.toURI().toURL();
+ }
+ }
+
+ // create and return the class loader
+ this.cl = new ExternalLibraryClassLoader(urls, parentClassLoader);
+
+ }
+
+ @Override
+ public ExternalFunctionLanguage getLanguage() {
+ return ExternalFunctionLanguage.JAVA;
+ }
+
+ public ClassLoader getClassLoader() {
+ return cl;
+ }
+
+ public void close() {
+ try {
+ if (cl != null) {
+ cl.close();
+ }
+ } catch (IOException e) {
+ LOGGER.error("Couldn't close classloader", e);
+ }
+ }
+
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibrary.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibrary.java
new file mode 100644
index 0000000..3ce3e91
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibrary.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.asterix.external.library;
+
+import java.io.File;
+
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+import org.apache.asterix.common.library.ILibrary;
+
+public class PythonLibrary implements ILibrary {
+
+ private final File path;
+
+ PythonLibrary(String path) {
+ this.path = new File(path);
+ }
+
+ @Override
+ public ExternalFunctionLanguage getLanguage() {
+ return ExternalFunctionLanguage.PYTHON;
+ }
+
+ public File getFile() {
+ return path;
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectAccessors.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectAccessors.java
index a7b97cf..230627f 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectAccessors.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectAccessors.java
@@ -27,7 +27,6 @@
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.dataflow.data.nontagged.serde.ABooleanSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.ACircleSerializerDeserializer;
import org.apache.asterix.dataflow.data.nontagged.serde.ADateSerializerDeserializer;
import org.apache.asterix.dataflow.data.nontagged.serde.ADateTimeSerializerDeserializer;
import org.apache.asterix.dataflow.data.nontagged.serde.ADoubleSerializerDeserializer;
@@ -38,11 +37,6 @@
import org.apache.asterix.dataflow.data.nontagged.serde.AInt64SerializerDeserializer;
import org.apache.asterix.dataflow.data.nontagged.serde.AInt8SerializerDeserializer;
import org.apache.asterix.dataflow.data.nontagged.serde.AIntervalSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.ALineSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.APoint3DSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.APointSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.APolygonSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.ARectangleSerializerDeserializer;
import org.apache.asterix.dataflow.data.nontagged.serde.ATimeSerializerDeserializer;
import org.apache.asterix.external.api.IJListAccessor;
import org.apache.asterix.external.api.IJObject;
@@ -51,7 +45,6 @@
import org.apache.asterix.external.library.TypeInfo;
import org.apache.asterix.external.library.java.base.JBoolean;
import org.apache.asterix.external.library.java.base.JByte;
-import org.apache.asterix.external.library.java.base.JCircle;
import org.apache.asterix.external.library.java.base.JDate;
import org.apache.asterix.external.library.java.base.JDateTime;
import org.apache.asterix.external.library.java.base.JDouble;
@@ -59,25 +52,15 @@
import org.apache.asterix.external.library.java.base.JFloat;
import org.apache.asterix.external.library.java.base.JInt;
import org.apache.asterix.external.library.java.base.JInterval;
-import org.apache.asterix.external.library.java.base.JLine;
import org.apache.asterix.external.library.java.base.JList;
import org.apache.asterix.external.library.java.base.JLong;
import org.apache.asterix.external.library.java.base.JOrderedList;
-import org.apache.asterix.external.library.java.base.JPoint;
-import org.apache.asterix.external.library.java.base.JPoint3D;
-import org.apache.asterix.external.library.java.base.JPolygon;
import org.apache.asterix.external.library.java.base.JRecord;
-import org.apache.asterix.external.library.java.base.JRectangle;
+import org.apache.asterix.external.library.java.base.JShort;
import org.apache.asterix.external.library.java.base.JString;
import org.apache.asterix.external.library.java.base.JTime;
import org.apache.asterix.external.library.java.base.JUnorderedList;
-import org.apache.asterix.om.base.ACircle;
import org.apache.asterix.om.base.ADuration;
-import org.apache.asterix.om.base.ALine;
-import org.apache.asterix.om.base.APoint;
-import org.apache.asterix.om.base.APoint3D;
-import org.apache.asterix.om.base.APolygon;
-import org.apache.asterix.om.base.ARectangle;
import org.apache.asterix.om.pointables.AFlatValuePointable;
import org.apache.asterix.om.pointables.AListVisitablePointable;
import org.apache.asterix.om.pointables.ARecordVisitablePointable;
@@ -131,15 +114,6 @@
case STRING:
accessor = new JStringAccessor();
break;
- case POINT:
- accessor = new JPointAccessor();
- break;
- case POINT3D:
- accessor = new JPoint3DAccessor();
- break;
- case LINE:
- accessor = new JLineAccessor();
- break;
case DATE:
accessor = new JDateAccessor();
break;
@@ -152,15 +126,6 @@
case INTERVAL:
accessor = new JIntervalAccessor();
break;
- case CIRCLE:
- accessor = new JCircleAccessor();
- break;
- case POLYGON:
- accessor = new JPolygonAccessor();
- break;
- case RECTANGLE:
- accessor = new JRectangleAccessor();
- break;
case TIME:
accessor = new JTimeAccessor();
break;
@@ -200,7 +165,7 @@
int s = pointable.getStartOffset();
short i = AInt16SerializerDeserializer.getShort(b, s + 1);
IJObject jObject = objectPool.allocate(BuiltinType.AINT16);
- ((JInt) jObject).setValue(i);
+ ((JShort) jObject).setValue(i);
return null;
}
}
@@ -389,106 +354,6 @@
}
}
- // Spatial Types
-
- public static class JCircleAccessor implements IJObjectAccessor {
-
- @Override
- public IJObject access(IPointable pointable, IObjectPool<IJObject, IAType> objectPool)
- throws HyracksDataException {
- byte[] b = pointable.getByteArray();
- int s = pointable.getStartOffset();
- int l = pointable.getLength();
- ACircle v = ACircleSerializerDeserializer.INSTANCE
- .deserialize(new DataInputStream(new ByteArrayInputStream(b, s + 1, l - 1)));
- JPoint jpoint = (JPoint) objectPool.allocate(BuiltinType.APOINT);
- jpoint.setValue(v.getP().getX(), v.getP().getY());
- IJObject jObject = objectPool.allocate(BuiltinType.ACIRCLE);
- ((JCircle) jObject).setValue(jpoint, v.getRadius());
- return jObject;
- }
- }
-
- public static class JPointAccessor implements IJObjectAccessor {
-
- @Override
- public IJObject access(IPointable pointable, IObjectPool<IJObject, IAType> objectPool)
- throws HyracksDataException {
- byte[] b = pointable.getByteArray();
- int s = pointable.getStartOffset();
- int l = pointable.getLength();
- APoint v = APointSerializerDeserializer.INSTANCE
- .deserialize(new DataInputStream(new ByteArrayInputStream(b, s + 1, l - 1)));
- JPoint jObject = (JPoint) objectPool.allocate(BuiltinType.APOINT);
- jObject.setValue(v.getX(), v.getY());
- return jObject;
- }
- }
-
- public static class JPoint3DAccessor implements IJObjectAccessor {
-
- @Override
- public IJObject access(IPointable pointable, IObjectPool<IJObject, IAType> objectPool)
- throws HyracksDataException {
- byte[] b = pointable.getByteArray();
- int s = pointable.getStartOffset();
- int l = pointable.getLength();
- APoint3D v = APoint3DSerializerDeserializer.INSTANCE
- .deserialize(new DataInputStream(new ByteArrayInputStream(b, s + 1, l - 1)));
- JPoint3D jObject = (JPoint3D) objectPool.allocate(BuiltinType.APOINT3D);
- jObject.setValue(v.getX(), v.getY(), v.getZ());
- return jObject;
- }
- }
-
- public static class JLineAccessor implements IJObjectAccessor {
-
- @Override
- public IJObject access(IPointable pointable, IObjectPool<IJObject, IAType> objectPool)
- throws HyracksDataException {
- byte[] b = pointable.getByteArray();
- int s = pointable.getStartOffset();
- int l = pointable.getLength();
- ALine v = ALineSerializerDeserializer.INSTANCE
- .deserialize(new DataInputStream(new ByteArrayInputStream(b, s + 1, l - 1)));
- JLine jObject = (JLine) objectPool.allocate(BuiltinType.ALINE);
- jObject.setValue(v.getP1(), v.getP2());
- return jObject;
- }
- }
-
- public static class JPolygonAccessor implements IJObjectAccessor {
-
- @Override
- public IJObject access(IPointable pointable, IObjectPool<IJObject, IAType> objectPool)
- throws HyracksDataException {
- byte[] b = pointable.getByteArray();
- int s = pointable.getStartOffset();
- int l = pointable.getLength();
- APolygon v = APolygonSerializerDeserializer.INSTANCE
- .deserialize(new DataInputStream(new ByteArrayInputStream(b, s + 1, l - 1)));
- JPolygon jObject = (JPolygon) objectPool.allocate(BuiltinType.APOLYGON);
- jObject.setValue(v.getPoints());
- return jObject;
- }
- }
-
- public static class JRectangleAccessor implements IJObjectAccessor {
-
- @Override
- public IJObject access(IPointable pointable, IObjectPool<IJObject, IAType> objectPool)
- throws HyracksDataException {
- byte[] b = pointable.getByteArray();
- int s = pointable.getStartOffset();
- int l = pointable.getLength();
- ARectangle v = ARectangleSerializerDeserializer.INSTANCE
- .deserialize(new DataInputStream(new ByteArrayInputStream(b, s + 1, l - 1)));
- JRectangle jObject = (JRectangle) objectPool.allocate(BuiltinType.ARECTANGLE);
- jObject.setValue(v.getP1(), v.getP2());
- return jObject;
- }
- }
-
public static class JRecordAccessor implements IJRecordAccessor {
private final TypeInfo typeInfo;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JBoolean.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JBoolean.java
index 428e0f9..d8716c4 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JBoolean.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JBoolean.java
@@ -28,7 +28,7 @@
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public final class JBoolean extends JObject {
+public final class JBoolean extends JObject<Boolean> {
private boolean aBoolean;
@@ -44,6 +44,10 @@
return aBoolean;
}
+ public Boolean getValueGeneric() {
+ return aBoolean;
+ }
+
@Override
public IAType getIAType() {
return BuiltinType.ABOOLEAN;
@@ -55,6 +59,11 @@
}
@Override
+ public void setValueGeneric(Boolean b) {
+ setValue(b);
+ }
+
+ @Override
public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.BOOLEAN);
ABooleanSerializerDeserializer.INSTANCE.serialize((ABoolean) getIAObject(), dataOutput);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JByte.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JByte.java
index 3f2afcd..49f4460 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JByte.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JByte.java
@@ -28,7 +28,7 @@
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public final class JByte extends JObject {
+public final class JByte extends JObject<Byte> {
public JByte(byte value) {
super(new AMutableInt8(value));
@@ -43,6 +43,11 @@
}
@Override
+ public Byte getValueGeneric() {
+ return ((AMutableInt8) value).getByteValue();
+ }
+
+ @Override
public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.TINYINT);
AInt8SerializerDeserializer.INSTANCE.serialize((AInt8) value, dataOutput);
@@ -57,4 +62,10 @@
public IAType getIAType() {
return BuiltinType.AINT8;
}
+
+ @Override
+ public void setValueGeneric(Byte o) {
+ setValue(o);
+
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JCircle.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JCircle.java
deleted file mode 100644
index 6c673a6..0000000
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JCircle.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.library.java.base;
-
-import java.io.DataOutput;
-
-import org.apache.asterix.dataflow.data.nontagged.serde.ACircleSerializerDeserializer;
-import org.apache.asterix.om.base.AMutableCircle;
-import org.apache.asterix.om.base.APoint;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.IAType;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public final class JCircle extends JObject {
-
- public JCircle(JPoint center, double radius) {
- super(new AMutableCircle((APoint) center.getIAObject(), radius));
- }
-
- public void setValue(JPoint center, double radius) {
- ((AMutableCircle) (value)).setValue((APoint) center.getIAObject(), radius);
- }
-
- public Pair<Double, Double> getCenter() {
- return Pair.of(((AMutableCircle) (value)).getP().getX(), ((AMutableCircle) (value)).getP().getY());
- }
-
- public double getRaidus() {
- return ((AMutableCircle) (value)).getRadius();
- }
-
- @Override
- public IAType getIAType() {
- return BuiltinType.ACIRCLE;
- }
-
- @Override
- public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
- serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.CIRCLE);
- ACircleSerializerDeserializer.INSTANCE.serialize((AMutableCircle) value, dataOutput);
- }
-
- @Override
- public void reset() {
- ((AMutableCircle) value).setValue(null, 0);
- }
-}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JComplexObject.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JComplexObject.java
new file mode 100644
index 0000000..a8aa153
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JComplexObject.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.asterix.external.library.java.base;
+
+import org.apache.asterix.external.api.IJObject;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.util.container.IObjectPool;
+
+public abstract class JComplexObject<T> implements IJObject<T> {
+
+ protected IObjectPool<IJObject, IAType> pool;
+
+ public void setPool(IObjectPool<IJObject, IAType> pool) {
+ this.pool = pool;
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JDate.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JDate.java
index a427688..3edf616 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JDate.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JDate.java
@@ -27,7 +27,7 @@
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public final class JDate extends JObject {
+public final class JDate extends JObject<Integer> {
public JDate(int chrononTimeInDays) {
super(new AMutableDate(chrononTimeInDays));
@@ -41,6 +41,10 @@
return ((AMutableDate) value).getChrononTimeInDays();
}
+ public Integer getValueGeneric() {
+ return getValue();
+ }
+
@Override
public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.DATE);
@@ -56,4 +60,10 @@
public IAType getIAType() {
return BuiltinType.ADATE;
}
+
+ @Override
+ public void setValueGeneric(Integer o) {
+ setValue(o);
+ }
+
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JDateTime.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JDateTime.java
index bafc31d..55f4ae2 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JDateTime.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JDateTime.java
@@ -19,6 +19,8 @@
package org.apache.asterix.external.library.java.base;
import java.io.DataOutput;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
import org.apache.asterix.dataflow.data.nontagged.serde.ADateTimeSerializerDeserializer;
import org.apache.asterix.om.base.AMutableDateTime;
@@ -27,7 +29,7 @@
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public final class JDateTime extends JObject {
+public final class JDateTime extends JObject<LocalDateTime> {
public JDateTime(long chrononTime) {
super(new AMutableDateTime(chrononTime));
@@ -41,6 +43,10 @@
return ((AMutableDateTime) value).getChrononTime();
}
+ public LocalDateTime getValueGeneric() {
+ return LocalDateTime.ofEpochSecond(((AMutableDateTime) value).getChrononTime(), 0, ZoneOffset.UTC);
+ }
+
@Override
public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.DATETIME);
@@ -56,4 +62,9 @@
public IAType getIAType() {
return BuiltinType.ADATETIME;
}
+
+ @Override
+ public void setValueGeneric(LocalDateTime dt) {
+ setValue(dt.toEpochSecond(ZoneOffset.UTC));
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JDouble.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JDouble.java
index 052dd91..4e9f783 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JDouble.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JDouble.java
@@ -28,7 +28,7 @@
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public final class JDouble extends JObject {
+public final class JDouble extends JObject<Double> {
public JDouble(double v) {
super(new AMutableDouble(v));
@@ -38,6 +38,10 @@
((AMutableDouble) value).setValue(v);
}
+ public Double getValueGeneric() {
+ return getValue();
+ }
+
public double getValue() {
return ((AMutableDouble) value).getDoubleValue();
}
@@ -57,4 +61,10 @@
public IAType getIAType() {
return BuiltinType.ADOUBLE;
}
+
+ @Override
+ public void setValueGeneric(Double o) {
+ setValue(o);
+ }
+
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JDuration.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JDuration.java
index 1b49624..c1c9832 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JDuration.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JDuration.java
@@ -19,16 +19,17 @@
package org.apache.asterix.external.library.java.base;
import java.io.DataOutput;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
import org.apache.asterix.dataflow.data.nontagged.serde.ADurationSerializerDeserializer;
import org.apache.asterix.om.base.AMutableDuration;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public final class JDuration extends JObject {
+public final class JDuration extends JObject<Duration> {
public JDuration(int months, long milliseconds) {
super(new AMutableDuration(months, milliseconds));
@@ -38,8 +39,10 @@
((AMutableDuration) value).setValue(months, milliseconds);
}
- public Pair<Integer, Long> getValue() {
- return Pair.of(((AMutableDuration) value).getMonths(), ((AMutableDuration) value).getMilliseconds());
+ public Duration getValueGeneric() {
+ int months = ((AMutableDuration) value).getMonths();
+ long millis = ((AMutableDuration) value).getMilliseconds();
+ return Duration.of(months, ChronoUnit.MONTHS).plusMillis(millis);
}
@Override
@@ -57,4 +60,14 @@
public IAType getIAType() {
return BuiltinType.ADURATION;
}
+
+ @Override
+ public void setValueGeneric(Duration o) {
+ long months = o.get(ChronoUnit.MONTHS);
+ if (months > Integer.MAX_VALUE) {
+ throw new ArithmeticException("Overflow");
+ }
+ long ms = o.minus(Duration.of(months, ChronoUnit.MONTHS)).get(ChronoUnit.MILLIS);
+ setValue((int) months, ms);
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JFloat.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JFloat.java
index 3d6ea6b..01efca9 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JFloat.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JFloat.java
@@ -28,16 +28,20 @@
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public final class JFloat extends JObject {
+public final class JFloat extends JObject<Float> {
public JFloat(float v) {
super(new AMutableFloat(v));
}
- public void setValue(float v) {
+ public void setValue(Float v) {
((AMutableFloat) value).setValue(v);
}
+ public Float getValueGeneric() {
+ return getValue();
+ }
+
public float getValue() {
return ((AMutableFloat) value).getFloatValue();
}
@@ -57,4 +61,10 @@
public IAType getIAType() {
return BuiltinType.AFLOAT;
}
+
+ @Override
+ public void setValueGeneric(Float o) {
+ setValue(o);
+ }
+
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JInt.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JInt.java
index f81a20a..bf45d8e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JInt.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JInt.java
@@ -28,7 +28,11 @@
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public class JInt extends JObject {
+public class JInt extends JObject<Integer> {
+
+ public JInt() {
+ this(-1);
+ }
public JInt(int value) {
super(new AMutableInt32(value));
@@ -43,6 +47,16 @@
}
@Override
+ public void setValueGeneric(Integer v) {
+ setValue(v);
+ }
+
+ @Override
+ public Integer getValueGeneric() {
+ return getValue();
+ }
+
+ @Override
public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.INTEGER);
AInt32SerializerDeserializer.INSTANCE.serialize((AInt32) value, dataOutput);
@@ -57,4 +71,5 @@
public IAType getIAType() {
return BuiltinType.AINT32;
}
+
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JInterval.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JInterval.java
index 1885a18..14caa1e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JInterval.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JInterval.java
@@ -19,15 +19,18 @@
package org.apache.asterix.external.library.java.base;
import java.io.DataOutput;
+import java.util.Arrays;
+import java.util.List;
import org.apache.asterix.dataflow.data.nontagged.serde.AIntervalSerializerDeserializer;
import org.apache.asterix.om.base.AMutableInterval;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public final class JInterval extends JObject {
+public final class JInterval extends JObject<List<Long>> {
public JInterval(long intervalStart, long intervalEnd) {
super(new AMutableInterval(intervalStart, intervalEnd, (byte) 0));
@@ -64,4 +67,21 @@
public IAType getIAType() {
return BuiltinType.AINTERVAL;
}
+
+ @Override
+ public void setValueGeneric(List<Long> o) {
+ try {
+ setValue(o.get(0), o.get(1), o.get(2).byteValue());
+ } catch (HyracksDataException e) {
+ throw new ArithmeticException("Invalid interval");
+ }
+ }
+
+ @Override
+ public List<Long> getValueGeneric() {
+ long type = getIntervalType();
+ Long[] interval = ArrayUtils.toObject(new long[] { getIntervalStart(), getIntervalEnd(), type });
+ return (Arrays.asList(interval));
+ }
+
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JLine.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JLine.java
deleted file mode 100644
index b471385..0000000
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JLine.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.library.java.base;
-
-import java.io.DataOutput;
-
-import org.apache.asterix.dataflow.data.nontagged.serde.ALineSerializerDeserializer;
-import org.apache.asterix.om.base.AMutableLine;
-import org.apache.asterix.om.base.APoint;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.IAType;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public final class JLine extends JObject {
-
- public JLine(JPoint p1, JPoint p2) {
- super(new AMutableLine((APoint) p1.getIAObject(), (APoint) p2.getIAObject()));
- }
-
- public void setValue(JPoint p1, JPoint p2) {
- ((AMutableLine) value).setValue((APoint) p1.getIAObject(), (APoint) p2.getIAObject());
- }
-
- public void setValue(APoint p1, APoint p2) {
- ((AMutableLine) value).setValue(p1, p2);
- }
-
- public Pair<Double, Double> getBeginPoint() {
- return Pair.of(((AMutableLine) value).getP1().getX(), ((AMutableLine) value).getP1().getY());
- }
-
- public Pair<Double, Double> getEndPoint() {
- return Pair.of(((AMutableLine) value).getP2().getX(), ((AMutableLine) value).getP2().getY());
- }
-
- @Override
- public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
- serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.LINE);
- ALineSerializerDeserializer.INSTANCE.serialize((AMutableLine) value, dataOutput);
- }
-
- @Override
- public void reset() {
- ((AMutableLine) value).setValue(null, null);
- }
-
- @Override
- public IAType getIAType() {
- return BuiltinType.ALINE;
- }
-}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JList.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JList.java
index 3478ed0..c12b4a2 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JList.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JList.java
@@ -18,18 +18,16 @@
*/
package org.apache.asterix.external.library.java.base;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
-import java.util.List;
import org.apache.asterix.external.api.IJObject;
-public abstract class JList implements IJObject {
- protected List<IJObject> jObjects;
+public abstract class JList<T> extends JComplexObject<T> {
+
+ protected Collection<IJObject> jObjects;
public JList() {
- jObjects = new ArrayList<>();
}
public boolean isEmpty() {
@@ -48,10 +46,6 @@
jObjects.clear();
}
- public IJObject getElement(int index) {
- return jObjects.get(index);
- }
-
public int size() {
return jObjects.size();
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JLong.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JLong.java
index c92b04c..8a6ad82 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JLong.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JLong.java
@@ -28,14 +28,18 @@
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public final class JLong extends JObject {
+public final class JLong extends JObject<Long> {
+
+ public JLong() {
+ this(0l);
+ }
public JLong(long v) {
super(new AMutableInt64(v));
}
- public void setValue(long v) {
- ((AMutableInt64) value).setValue(v);
+ public void setValue(long l) {
+ ((AMutableInt64) value).setValue(l);
}
public long getValue() {
@@ -43,6 +47,16 @@
}
@Override
+ public void setValueGeneric(Long v) {
+ ((AMutableInt64) value).setValue(v);
+ }
+
+ @Override
+ public Long getValueGeneric() {
+ return ((AMutableInt64) value).getLongValue();
+ }
+
+ @Override
public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.BIGINT);
AInt64SerializerDeserializer.INSTANCE.serialize((AInt64) value, dataOutput);
@@ -57,4 +71,5 @@
public IAType getIAType() {
return BuiltinType.AINT64;
}
+
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JMissing.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JMissing.java
index 33c17b0..6c814b3 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JMissing.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JMissing.java
@@ -27,7 +27,7 @@
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public final class JMissing extends JObject {
+public final class JMissing extends JObject<Object> {
public final static JMissing INSTANCE = new JMissing();
@@ -42,6 +42,18 @@
}
@Override
+ public void setValueGeneric(Object o) {
+ if (o != null) {
+ throw new IllegalArgumentException("Not null");
+ }
+ }
+
+ @Override
+ public Object getValueGeneric() {
+ return null;
+ }
+
+ @Override
public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.MISSING);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JNull.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JNull.java
index 885f74e..e79cd9b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JNull.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JNull.java
@@ -47,6 +47,18 @@
}
@Override
+ public void setValueGeneric(Object o) {
+ if (o != null) {
+ throw new IllegalArgumentException("Not null");
+ }
+ }
+
+ @Override
+ public Object getValueGeneric() {
+ return null;
+ }
+
+ @Override
public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.NULL);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JObject.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JObject.java
index 9ba7cd1..8dce4ce 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JObject.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JObject.java
@@ -18,20 +18,51 @@
*/
package org.apache.asterix.external.library.java.base;
+import static org.apache.asterix.om.types.AOrderedListType.FULL_OPEN_ORDEREDLIST_TYPE;
+import static org.apache.asterix.om.types.AUnorderedListType.FULLY_OPEN_UNORDEREDLIST_TYPE;
+import static org.apache.asterix.om.utils.RecordUtil.FULLY_OPEN_RECORD_TYPE;
+
import java.io.DataOutput;
import java.io.IOException;
+import java.time.Duration;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.asterix.external.api.IJObject;
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.util.container.IObjectPool;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public abstract class JObject implements IJObject {
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Multiset;
+public abstract class JObject<T> implements IJObject<T> {
+
+ private static final Map<Class, IAType> typeConv = new ImmutableMap.Builder<Class, IAType>()
+ .put(HashMap.class, FULLY_OPEN_RECORD_TYPE).put(Byte.class, BuiltinType.AINT8)
+ .put(Short.class, BuiltinType.AINT16).put(Integer.class, BuiltinType.AINT32)
+ .put(Long.class, BuiltinType.AINT64).put(Float.class, BuiltinType.AFLOAT)
+ .put(Double.class, BuiltinType.ADOUBLE).put(LocalTime.class, BuiltinType.ATIME)
+ .put(LocalDate.class, BuiltinType.ADATE).put(LocalDateTime.class, BuiltinType.ADATETIME)
+ .put(Duration.class, BuiltinType.ADURATION).put(List.class, FULL_OPEN_ORDEREDLIST_TYPE)
+ .put(String.class, BuiltinType.ASTRING).put(Multiset.class, FULLY_OPEN_UNORDEREDLIST_TYPE).build();
protected IAObject value;
protected byte[] bytes;
+ protected IObjectPool<IJObject, Class> pool;
protected JObject() {
+
+ }
+
+ public static IAType convertType(Class clazz) {
+ return typeConv.get(clazz);
}
protected JObject(IAObject value) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JOrderedList.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JOrderedList.java
index b2c2b21..07c7ea0 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JOrderedList.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JOrderedList.java
@@ -19,6 +19,7 @@
package org.apache.asterix.external.library.java.base;
import java.io.DataOutput;
+import java.util.ArrayList;
import java.util.List;
import org.apache.asterix.builders.IAsterixListBuilder;
@@ -32,22 +33,32 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-public final class JOrderedList extends JList {
+public final class JOrderedList extends JList<List<? extends Object>> {
private AOrderedListType listType;
+ public JOrderedList() {
+ this(AOrderedListType.FULL_OPEN_ORDEREDLIST_TYPE);
+ }
+
public JOrderedList(IJType listItemType) {
super();
+ jObjects = new ArrayList<>();
this.listType = new AOrderedListType(listItemType.getIAType(), null);
}
public JOrderedList(IAType listItemType) {
super();
+ jObjects = new ArrayList<>();
this.listType = new AOrderedListType(listItemType, null);
}
+ public List<? extends Object> getValueGeneric() {
+ return (ArrayList) jObjects;
+ }
+
public List<IJObject> getValue() {
- return jObjects;
+ return (List) jObjects;
}
@Override
@@ -65,13 +76,33 @@
}
@Override
+ public void setValueGeneric(List<? extends Object> vals) throws HyracksDataException {
+ reset();
+ if (vals.size() > 0) {
+ Object first = vals.get(0);
+ IAType asxClass = JObject.convertType(first.getClass());
+ IJObject obj = pool.allocate(asxClass);
+ obj.setValueGeneric(first);
+ IAType listType = obj.getIAType();
+ this.listType = new AOrderedListType(listType, "");
+ }
+ for (Object v : vals) {
+ IAType asxClass = JObject.convertType(v.getClass());
+ IJObject obj = pool.allocate(asxClass);
+ obj.setValueGeneric(v);
+ add(obj);
+ }
+
+ }
+
+ @Override
public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
IAsterixListBuilder listBuilder = new OrderedListBuilder();
listBuilder.reset(listType);
ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
for (IJObject jObject : jObjects) {
fieldValue.reset();
- jObject.serialize(fieldValue.getDataOutput(), true);
+ jObject.serialize(fieldValue.getDataOutput(), writeTypeTag);
listBuilder.addItem(fieldValue);
}
listBuilder.write(dataOutput, writeTypeTag);
@@ -82,4 +113,5 @@
public void reset() {
jObjects.clear();
}
+
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JPoint.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JPoint.java
deleted file mode 100644
index 9b1b8e3..0000000
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JPoint.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.library.java.base;
-
-import java.io.DataOutput;
-
-import org.apache.asterix.dataflow.data.nontagged.serde.APointSerializerDeserializer;
-import org.apache.asterix.om.base.AMutablePoint;
-import org.apache.asterix.om.base.APoint;
-import org.apache.asterix.om.base.IAObject;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.IAType;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public final class JPoint extends JObject {
-
- public JPoint(double x, double y) {
- super(new AMutablePoint(x, y));
- }
-
- public void setValue(double x, double y) {
- ((AMutablePoint) value).setValue(x, y);
- }
-
- public double getXValue() {
- return ((AMutablePoint) value).getX();
- }
-
- public double getYValue() {
- return ((AMutablePoint) value).getY();
- }
-
- public IAObject getValue() {
- return value;
- }
-
- @Override
- public String toString() {
- return value.toString();
- }
-
- @Override
- public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
- serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.POINT);
- APointSerializerDeserializer.INSTANCE.serialize((APoint) value, dataOutput);
- }
-
- @Override
- public void reset() {
- ((AMutablePoint) value).setValue(0, 0);
- }
-
- @Override
- public IAType getIAType() {
- return BuiltinType.APOINT;
- }
-}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JPoint3D.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JPoint3D.java
deleted file mode 100644
index c2c1917..0000000
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JPoint3D.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.library.java.base;
-
-import java.io.DataOutput;
-
-import org.apache.asterix.dataflow.data.nontagged.serde.APoint3DSerializerDeserializer;
-import org.apache.asterix.om.base.AMutablePoint3D;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.IAType;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public final class JPoint3D extends JObject {
-
- public JPoint3D(double x, double y, double z) {
- super(new AMutablePoint3D(x, y, z));
- }
-
- public void setValue(double x, double y, double z) {
- ((AMutablePoint3D) value).setValue(x, y, z);
- }
-
- public double getXValue() {
- return ((AMutablePoint3D) value).getX();
- }
-
- public double getYValue() {
- return ((AMutablePoint3D) value).getY();
- }
-
- public double getZValue() {
- return ((AMutablePoint3D) value).getZ();
- }
-
- @Override
- public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
- serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.POINT3D);
- APoint3DSerializerDeserializer.INSTANCE.serialize((AMutablePoint3D) value, dataOutput);
- }
-
- @Override
- public void reset() {
- ((AMutablePoint3D) value).setValue(0, 0, 0);
- }
-
- @Override
- public IAType getIAType() {
- return BuiltinType.APOINT3D;
- }
-}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JPolygon.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JPolygon.java
deleted file mode 100644
index 1ec9cae..0000000
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JPolygon.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.library.java.base;
-
-import java.io.DataOutput;
-
-import org.apache.asterix.dataflow.data.nontagged.serde.APolygonSerializerDeserializer;
-import org.apache.asterix.om.base.AMutablePolygon;
-import org.apache.asterix.om.base.APoint;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.IAType;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public final class JPolygon extends JObject {
-
- public JPolygon(JPoint[] points) {
- super(new AMutablePolygon(getAPoints(points)));
- }
-
- public void setValue(APoint[] points) {
- ((AMutablePolygon) value).setValue(points);
- }
-
- public void setValue(JPoint[] points) {
- ((AMutablePolygon) value).setValue(getAPoints(points));
- }
-
- public APoint[] getValue() {
- return ((AMutablePolygon) value).getPoints();
- }
-
- @Override
- public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
- serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.POLYGON);
- APolygonSerializerDeserializer.INSTANCE.serialize((AMutablePolygon) value, dataOutput);
- }
-
- @Override
- public void reset() {
- ((AMutablePolygon) value).setValue(null);
- }
-
- protected static APoint[] getAPoints(JPoint[] jpoints) {
- APoint[] apoints = new APoint[jpoints.length];
- int index = 0;
- for (JPoint jpoint : jpoints) {
- apoints[index++] = (APoint) jpoint.getIAObject();
- }
- return apoints;
- }
-
- @Override
- public IAType getIAType() {
- return BuiltinType.APOLYGON;
- }
-}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JRecord.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JRecord.java
index 205e228..9bd6461 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JRecord.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JRecord.java
@@ -18,8 +18,11 @@
*/
package org.apache.asterix.external.library.java.base;
+import static org.apache.asterix.om.utils.RecordUtil.FULLY_OPEN_RECORD_TYPE;
+
import java.io.DataOutput;
import java.io.IOException;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -38,7 +41,7 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-public final class JRecord implements IJObject {
+public final class JRecord extends JComplexObject<Map<String, Object>> {
private static final AStringSerializerDeserializer aStringSerDer = AStringSerializerDeserializer.INSTANCE;
private ARecordType recordType;
@@ -49,6 +52,12 @@
ArrayBackedValueStorage fieldValueBuffer = new ArrayBackedValueStorage();
AMutableString nameString = new AMutableString("");
+ public JRecord() {
+ this.recordType = FULLY_OPEN_RECORD_TYPE;
+ this.fields = new IJObject[] {};
+ this.openFields = new LinkedHashMap<>();
+ }
+
public JRecord(ARecordType recordType, IJObject[] fields) {
this.recordType = recordType;
this.fields = fields;
@@ -138,7 +147,7 @@
nameString.setValue(entry.getKey());
fieldNameBuffer.getDataOutput().write(ATypeTag.STRING.serialize());
aStringSerDer.serialize(nameString, fieldNameBuffer.getDataOutput());
- entry.getValue().serialize(fieldValueBuffer.getDataOutput(), true);
+ entry.getValue().serialize(fieldValueBuffer.getDataOutput(), writeTypeTag);
recordBuilder.addField(fieldNameBuffer, fieldValueBuffer);
}
}
@@ -173,6 +182,30 @@
}
@Override
+ public void setValueGeneric(Map<String, Object> o) throws HyracksDataException {
+ reset();
+ for (Map.Entry<String, Object> e : o.entrySet()) {
+ IAType asxClass = JObject.convertType(e.getValue().getClass());
+ IJObject obj = pool.allocate(asxClass);
+ obj.setValueGeneric(e.getValue());
+ openFields.put(e.getKey(), obj);
+ }
+ }
+
+ @Override
+ public Map<String, Object> getValueGeneric() {
+ HashMap<String, Object> rec = new HashMap<>();
+ String[] closedFieldNames = recordType.getFieldNames();
+ int idx = 0;
+ for (IJObject j : fields) {
+ rec.put(closedFieldNames[idx], j.getValueGeneric());
+ idx++;
+ }
+ openFields.entrySet().forEach(e -> rec.put(e.getKey(), e.getValue().getValueGeneric()));
+ return rec;
+ }
+
+ @Override
public void reset() throws HyracksDataException {
if (openFields != null && !openFields.isEmpty()) {
openFields.clear();
@@ -191,4 +224,5 @@
this.fields = fields;
this.openFields = openFields;
}
+
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JRectangle.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JRectangle.java
deleted file mode 100644
index a07896f..0000000
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JRectangle.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.library.java.base;
-
-import java.io.DataOutput;
-
-import org.apache.asterix.dataflow.data.nontagged.serde.ARectangleSerializerDeserializer;
-import org.apache.asterix.om.base.AMutableRectangle;
-import org.apache.asterix.om.base.APoint;
-import org.apache.asterix.om.base.ARectangle;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.IAType;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public final class JRectangle extends JObject {
-
- public JRectangle(JPoint p1, JPoint p2) {
- super(new AMutableRectangle((APoint) p1.getIAObject(), (APoint) p2.getIAObject()));
- }
-
- public void setValue(JPoint p1, JPoint p2) {
- ((AMutableRectangle) value).setValue((APoint) p1.getValue(), (APoint) p2.getValue());
- }
-
- public void setValue(APoint p1, APoint p2) {
- ((AMutableRectangle) value).setValue(p1, p2);
- }
-
- public ARectangle getValue() {
- return (AMutableRectangle) value;
- }
-
- public IAType getIAType() {
- return BuiltinType.ARECTANGLE;
- }
-
- @Override
- public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
- serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.RECTANGLE);
- ARectangleSerializerDeserializer.INSTANCE.serialize((ARectangle) value, dataOutput);
- }
-
- @Override
- public void reset() {
- ((AMutableRectangle) value).setValue(null, null);
- }
-}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JShort.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JShort.java
index 9cec96e..ce4ac5d 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JShort.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JShort.java
@@ -28,7 +28,7 @@
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public final class JShort extends JObject {
+public final class JShort extends JObject<Short> {
public JShort(short value) {
super(new AMutableInt16(value));
@@ -42,6 +42,10 @@
return ((AMutableInt16) value).getShortValue();
}
+ public Short getValueGeneric() {
+ return ((AMutableInt16) value).getShortValue();
+ }
+
@Override
public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.SMALLINT);
@@ -57,4 +61,9 @@
public IAType getIAType() {
return BuiltinType.AINT16;
}
+
+ @Override
+ public void setValueGeneric(Short s) {
+ setValueGeneric(s);
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JString.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JString.java
index f8e44d4..082e697 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JString.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JString.java
@@ -28,7 +28,11 @@
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public final class JString extends JObject {
+public final class JString extends JObject<String> {
+
+ public JString() {
+ super(new AMutableString(""));
+ }
public JString(String v) {
super(new AMutableString(v));
@@ -42,6 +46,10 @@
return ((AMutableString) value).getStringValue();
}
+ public String getValueGeneric() {
+ return getValue();
+ }
+
@Override
public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.STRING);
@@ -57,4 +65,9 @@
public IAType getIAType() {
return BuiltinType.ASTRING;
}
+
+ @Override
+ public void setValueGeneric(String o) {
+ setValue(o);
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JTime.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JTime.java
index 5e672b0..2069e49 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JTime.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JTime.java
@@ -19,6 +19,7 @@
package org.apache.asterix.external.library.java.base;
import java.io.DataOutput;
+import java.time.LocalTime;
import org.apache.asterix.dataflow.data.nontagged.serde.ATimeSerializerDeserializer;
import org.apache.asterix.om.base.AMutableTime;
@@ -27,7 +28,7 @@
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public final class JTime extends JObject {
+public final class JTime extends JObject<LocalTime> {
public JTime(int timeInMillsec) {
super(new AMutableTime(timeInMillsec));
@@ -41,6 +42,10 @@
return ((AMutableTime) value).getChrononTime();
}
+ public LocalTime getValueGeneric() {
+ return LocalTime.ofSecondOfDay(((AMutableTime) value).getChrononTime());
+ }
+
@Override
public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
serializeTypeTag(writeTypeTag, dataOutput, ATypeTag.TIME);
@@ -56,4 +61,9 @@
public IAType getIAType() {
return BuiltinType.ATIME;
}
+
+ @Override
+ public void setValueGeneric(LocalTime t) {
+ setValue(t.toSecondOfDay());
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JUnorderedList.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JUnorderedList.java
index 070a2dc..3e41855 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JUnorderedList.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JUnorderedList.java
@@ -19,7 +19,6 @@
package org.apache.asterix.external.library.java.base;
import java.io.DataOutput;
-import java.util.List;
import org.apache.asterix.builders.IAsterixListBuilder;
import org.apache.asterix.builders.UnorderedListBuilder;
@@ -32,27 +31,27 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-public final class JUnorderedList extends JList {
+import com.google.common.collect.HashMultiset;
+import com.google.common.collect.Multiset;
+
+public final class JUnorderedList extends JList<Multiset<? extends Object>> {
private AUnorderedListType listType;
public JUnorderedList(IJType elementType) {
super();
+ jObjects = HashMultiset.create();
this.listType = new AUnorderedListType(elementType.getIAType(), null);
}
public JUnorderedList(IAType elementType) {
super();
+ jObjects = HashMultiset.create();
this.listType = new AUnorderedListType(elementType, null);
}
- public List<IJObject> getValue() {
- return jObjects;
- }
-
- @Override
- public void add(IJObject jObject) {
- jObjects.add(jObject);
+ public Multiset<? extends Object> getValueGeneric() {
+ return (Multiset) jObjects;
}
@Override
@@ -70,13 +69,27 @@
}
@Override
+ public void setValueGeneric(Multiset<? extends Object> vals) throws HyracksDataException {
+ reset();
+ for (Object v : vals) {
+ IAType asxClass = JObject.convertType(v.getClass());
+ IJObject obj = pool.allocate(asxClass);
+ if (this.listType == null) {
+ this.listType = new AUnorderedListType(obj.getIAType(), "");
+ }
+ obj.setValueGeneric(v);
+ add(obj);
+ }
+ }
+
+ @Override
public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
IAsterixListBuilder listBuilder = new UnorderedListBuilder();
listBuilder.reset(listType);
ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
for (IJObject jObject : jObjects) {
fieldValue.reset();
- jObject.serialize(fieldValue.getDataOutput(), true);
+ jObject.serialize(fieldValue.getDataOutput(), writeTypeTag);
listBuilder.addItem(fieldValue);
}
listBuilder.write(dataOutput, writeTypeTag);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
index 7c61653..3b2d95f 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
@@ -24,10 +24,13 @@
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+import org.apache.asterix.common.library.ILibrary;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.external.feed.api.IFeed;
import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.library.JavaLibrary;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
@@ -105,7 +108,12 @@
INcApplicationContext runtimeCtx =
(INcApplicationContext) ctx.getJobletContext().getServiceContext().getApplicationContext();
ILibraryManager libraryManager = runtimeCtx.getLibraryManager();
- ClassLoader classLoader = libraryManager.getLibraryClassLoader(feedId.getDataverseName(), adaptorLibraryName);
+
+ ILibrary lib = libraryManager.getLibrary(feedId.getDataverseName(), adaptorLibraryName);
+ if (lib.getLanguage() != ExternalFunctionLanguage.JAVA) {
+ throw new HyracksDataException("Unexpected library language: " + lib.getLanguage());
+ }
+ ClassLoader classLoader = ((JavaLibrary) lib).getClassLoader();
if (classLoader != null) {
try {
adapterFactory = (IAdapterFactory) (classLoader.loadClass(adaptorFactoryClassName).newInstance());
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 93844d1..8aebd90 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -24,12 +24,15 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+import org.apache.asterix.common.library.ILibrary;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.external.api.IDataParserFactory;
import org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType;
import org.apache.asterix.external.api.IInputStreamFactory;
import org.apache.asterix.external.api.IRecordReaderFactory;
+import org.apache.asterix.external.library.JavaLibrary;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.AUnionType;
@@ -118,10 +121,6 @@
&& (aString.trim().length() > 1));
}
- public static ClassLoader getClassLoader(ILibraryManager libraryManager, DataverseName dataverse, String library) {
- return libraryManager.getLibraryClassLoader(dataverse, library);
- }
-
public static String getLibraryName(String aString) {
return aString.trim().split(FeedConstants.NamingConstants.LIBRARY_NAME_SEPARATOR)[0];
}
@@ -135,7 +134,11 @@
try {
String libraryName = getLibraryName(stream);
String className = getExternalClassName(stream);
- ClassLoader classLoader = getClassLoader(libraryManager, dataverse, libraryName);
+ ILibrary lib = libraryManager.getLibrary(dataverse, libraryName);
+ if (lib.getLanguage() != ExternalFunctionLanguage.JAVA) {
+ throw new HyracksDataException("Unexpected library language: " + lib.getLanguage());
+ }
+ ClassLoader classLoader = ((JavaLibrary) lib).getClassLoader();
return ((IInputStreamFactory) (classLoader.loadClass(className).newInstance()));
} catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
throw new RuntimeDataException(ErrorCode.UTIL_EXTERNAL_DATA_UTILS_FAIL_CREATE_STREAM_FACTORY, e);
@@ -233,7 +236,11 @@
}
DataverseName dataverseName = DataverseName.createSinglePartName(dataverseAndLibrary[0]); //TODO(MULTI_PART_DATAVERSE_NAME):REVISIT
String libraryName = dataverseAndLibrary[1];
- ClassLoader classLoader = libraryManager.getLibraryClassLoader(dataverseName, libraryName);
+ ILibrary lib = libraryManager.getLibrary(dataverseName, libraryName);
+ if (lib.getLanguage() != ExternalFunctionLanguage.JAVA) {
+ throw new AsterixException("Unexpected library language: " + lib.getLanguage());
+ }
+ ClassLoader classLoader = ((JavaLibrary) lib).getClassLoader();
try {
return (IRecordReaderFactory<?>) classLoader.loadClass(libraryAndFactory[1]).newInstance();
} catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
@@ -246,7 +253,11 @@
try {
String library = parserFactoryName.substring(0,
parserFactoryName.indexOf(ExternalDataConstants.EXTERNAL_LIBRARY_SEPARATOR));
- ClassLoader classLoader = libraryManager.getLibraryClassLoader(dataverse, library);
+ ILibrary lib = libraryManager.getLibrary(dataverse, library);
+ if (lib.getLanguage() != ExternalFunctionLanguage.JAVA) {
+ throw new AsterixException("Unexpected library language: " + lib.getLanguage());
+ }
+ ClassLoader classLoader = ((JavaLibrary) lib).getClassLoader();
return (IDataParserFactory) classLoader
.loadClass(parserFactoryName
.substring(parserFactoryName.indexOf(ExternalDataConstants.EXTERNAL_LIBRARY_SEPARATOR) + 1))
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/AllTypesFunction.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/AllTypesFunction.java
index 7e476e2..325e368 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/AllTypesFunction.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/AllTypesFunction.java
@@ -23,18 +23,13 @@
import org.apache.asterix.external.library.java.JBuiltinType;
import org.apache.asterix.external.library.java.JTypeTag;
import org.apache.asterix.external.library.java.base.JBoolean;
-import org.apache.asterix.external.library.java.base.JCircle;
import org.apache.asterix.external.library.java.base.JDate;
import org.apache.asterix.external.library.java.base.JDateTime;
import org.apache.asterix.external.library.java.base.JDouble;
import org.apache.asterix.external.library.java.base.JDuration;
import org.apache.asterix.external.library.java.base.JFloat;
import org.apache.asterix.external.library.java.base.JInt;
-import org.apache.asterix.external.library.java.base.JLine;
import org.apache.asterix.external.library.java.base.JOrderedList;
-import org.apache.asterix.external.library.java.base.JPoint;
-import org.apache.asterix.external.library.java.base.JPoint3D;
-import org.apache.asterix.external.library.java.base.JPolygon;
import org.apache.asterix.external.library.java.base.JRecord;
import org.apache.asterix.external.library.java.base.JString;
import org.apache.asterix.external.library.java.base.JTime;
@@ -45,7 +40,7 @@
private JOrderedList newFieldList;
@Override
- public void initialize(IFunctionHelper functionHelper) throws Exception {
+ public void initialize(IFunctionHelper functionHelper) {
newFieldList = new JOrderedList(JBuiltinType.JINT);
}
@@ -69,11 +64,6 @@
JTime time = (JTime) inputRecord.getValueByName("time");
JDateTime dateTime = (JDateTime) inputRecord.getValueByName("datetime");
JDuration duration = (JDuration) inputRecord.getValueByName("duration");
- JPoint location2d = (JPoint) inputRecord.getValueByName("location2d");
- JPoint3D location3d = (JPoint3D) inputRecord.getValueByName("location3d");
- JLine line = (JLine) inputRecord.getValueByName("line");
- JPolygon polygon = (JPolygon) inputRecord.getValueByName("polygon");
- JCircle circle = (JCircle) inputRecord.getValueByName("circle");
JRecord result = (JRecord) functionHelper.getResultObject();
result.setField("id", id);
@@ -91,22 +81,11 @@
result.setField("time", time);
result.setField("datetime", dateTime);
result.setField("duration", duration);
- result.setField("location2d", location2d);
- result.setField("location3d", location3d);
- result.setField("line", line);
- result.setField("polygon", polygon);
- result.setField("circle", circle);
JString newFieldString = (JString) functionHelper.getObject(JTypeTag.STRING);
newFieldString.setValue("processed");
result.addField("status", newFieldString);
- /*
- * JString element = (JString)
- * functionHelper.getObject(JTypeTag.STRING); element.setValue("raman");
- * newFieldList.add(element); result.addField("mylist", newFieldList);
- */
-
JString newFieldString2 = (JString) functionHelper.getObject(JTypeTag.STRING);
newFieldString2.setValue("this is working");
result.addField("working", newFieldString);
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/CapitalFinderFunction.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/CapitalFinderFunction.java
index 6219ad4..d2881e1 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/CapitalFinderFunction.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/CapitalFinderFunction.java
@@ -43,7 +43,6 @@
JRecord record = (JRecord) functionHelper.getResultObject();
String capitalCity = capitalList.getProperty(country.getValue(), NOT_FOUND);
capital.setValue(capitalCity);
-
record.setField("country", country);
record.setField("capital", capital);
functionHelper.setResult(record);
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/MyArraySumFunction.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/MyArraySumFunction.java
index 67c2889..9767e11 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/MyArraySumFunction.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/MyArraySumFunction.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.external.library;
+import java.util.List;
+
import org.apache.asterix.external.api.IExternalScalarFunction;
import org.apache.asterix.external.api.IFunctionHelper;
import org.apache.asterix.external.library.java.base.JInt;
@@ -35,9 +37,10 @@
@Override
public void evaluate(IFunctionHelper functionHelper) throws Exception {
JOrderedList arg0 = (JOrderedList) (functionHelper.getArgument(0));
+ List<JInt> arg = (List) arg0.getValue();
int sum = 0;
- for (int iter1 = 0; iter1 < arg0.size(); iter1++) {
- sum += ((JInt) arg0.getValue().get(iter1)).getValue();
+ for (int iter1 = 0; iter1 < arg.size(); iter1++) {
+ sum += arg.get(iter1).getValue();
}
result.setValue(sum);
functionHelper.setResult(result);
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/OpenCapitalFinderFunction.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/OpenCapitalFinderFunction.java
index 141aa10..095e9f2 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/OpenCapitalFinderFunction.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/OpenCapitalFinderFunction.java
@@ -44,7 +44,7 @@
JString country = ((JString) functionHelper.getArgument(0));
ARecordType recordType = new ARecordType("all", new String[] {}, new IAType[] {}, true);
JRecord record = (JRecord) functionHelper.getResultObject(recordType);
- String capitalCity = capitalList.getProperty(country.getValue(), NOT_FOUND);
+ String capitalCity = capitalList.getProperty(country.getValueGeneric(), NOT_FOUND);
capital.setValue(capitalCity);
record.setField("country", country);
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeValidationFunction.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeValidationFunction.java
index 93bffe2..95c58aa 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeValidationFunction.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/TypeValidationFunction.java
@@ -21,15 +21,11 @@
import org.apache.asterix.external.api.IExternalScalarFunction;
import org.apache.asterix.external.api.IFunctionHelper;
import org.apache.asterix.external.library.java.base.JBoolean;
-import org.apache.asterix.external.library.java.base.JCircle;
import org.apache.asterix.external.library.java.base.JDate;
import org.apache.asterix.external.library.java.base.JDateTime;
import org.apache.asterix.external.library.java.base.JDouble;
import org.apache.asterix.external.library.java.base.JFloat;
import org.apache.asterix.external.library.java.base.JInt;
-import org.apache.asterix.external.library.java.base.JLine;
-import org.apache.asterix.external.library.java.base.JPoint;
-import org.apache.asterix.external.library.java.base.JRectangle;
import org.apache.asterix.external.library.java.base.JString;
public class TypeValidationFunction implements IExternalScalarFunction {
@@ -48,12 +44,8 @@
JString stringVal = (JString) functionHelper.getArgument(2);
JDouble doubleVal = (JDouble) functionHelper.getArgument(3);
JBoolean booleanVal = (JBoolean) functionHelper.getArgument(4);
- JPoint pointVal = (JPoint) functionHelper.getArgument(5);
- JDate dateVal = (JDate) functionHelper.getArgument(6);
- JDateTime datetimeVal = (JDateTime) functionHelper.getArgument(7);
- JLine lineVal = (JLine) functionHelper.getArgument(8);
- JCircle circleVal = (JCircle) functionHelper.getArgument(9);
- JRectangle rectangleVal = (JRectangle) functionHelper.getArgument(10);
+ JDate dateVal = (JDate) functionHelper.getArgument(5);
+ JDateTime datetimeVal = (JDateTime) functionHelper.getArgument(6);
StringBuilder sb = new StringBuilder();
sb.append(int32.getIAObject() + " ");
@@ -61,12 +53,8 @@
sb.append(stringVal.getIAObject() + " ");
sb.append(doubleVal.getIAObject() + " ");
sb.append(booleanVal.getIAObject() + " ");
- sb.append(pointVal.getIAObject() + " ");
sb.append(dateVal.getIAObject() + " ");
sb.append(datetimeVal.getIAObject() + " ");
- sb.append(lineVal.getIAObject() + " ");
- sb.append(circleVal.getIAObject() + " ");
- sb.append(rectangleVal.getIAObject());
result.setValue(sb.toString());
functionHelper.setResult(result);
}
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/UpperCaseFunction.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/UpperCaseFunction.java
index 398504a..070690e 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/UpperCaseFunction.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/UpperCaseFunction.java
@@ -49,12 +49,11 @@
JOrderedList capList = new JOrderedList(BuiltinType.ASTRING);
JInt id = (JInt) inputRecord.getValueByName("id");
id.setValue(id.getValue() * -1);
-
for (int iter1 = 0; iter1 < textList.getValue().size(); iter1++) {
JRecord originalElement = (JRecord) textList.getValue().get(iter1);
JString originalText = (JString) originalElement.getValueByName("text");
JString capText = new JString(originalText.getValue().toUpperCase());
- capList.getValue().add(capText);
+ capList.add(capText);
}
JInt element_n = new JInt(textList.size());
JRecord result = (JRecord) functionHelper.getResultObject();
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
index 9635479..790d7dd 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
@@ -28,12 +28,15 @@
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.MetadataException;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+import org.apache.asterix.common.library.ILibrary;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.external.api.IDataSourceAdapter;
import org.apache.asterix.external.api.IDataSourceAdapter.AdapterType;
import org.apache.asterix.external.feed.api.IFeed;
import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.library.JavaLibrary;
import org.apache.asterix.external.provider.AdapterFactoryProvider;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
@@ -53,6 +56,7 @@
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
/**
* A utility class for providing helper functions for feeds TODO: Refactor this
@@ -129,8 +133,11 @@
case EXTERNAL:
String[] anameComponents = adapterName.split("#");
String libraryName = anameComponents[0];
- ClassLoader cl =
- appCtx.getLibraryManager().getLibraryClassLoader(feed.getDataverseName(), libraryName);
+ ILibrary lib = appCtx.getLibraryManager().getLibrary(feed.getDataverseName(), libraryName);
+ if (lib.getLanguage() != ExternalFunctionLanguage.JAVA) {
+ throw new HyracksDataException("Unexpected library language: " + lib.getLanguage());
+ }
+ ClassLoader cl = ((JavaLibrary) lib).getClassLoader();
adapterFactory = (IAdapterFactory) cl.loadClass(adapterFactoryClassname).newInstance();
break;
default:
@@ -202,8 +209,11 @@
case EXTERNAL:
String[] anameComponents = adapterName.split("#");
String libraryName = anameComponents[0];
- ClassLoader cl =
- appCtx.getLibraryManager().getLibraryClassLoader(feed.getDataverseName(), libraryName);
+ ILibrary lib = appCtx.getLibraryManager().getLibrary(feed.getDataverseName(), libraryName);
+ if (lib.getLanguage() != ExternalFunctionLanguage.JAVA) {
+ throw new HyracksDataException("Unexpected library language: " + lib.getLanguage());
+ }
+ ClassLoader cl = ((JavaLibrary) lib).getClassLoader();
adapterFactory = (IAdapterFactory) cl.loadClass(adapterFactoryClassname).newInstance();
break;
default:
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
index 537cc24..ced8dc0 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
@@ -24,10 +24,10 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.entities.Function;
-import org.apache.asterix.om.functions.ExternalFunctionLanguage;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
index c0ef0fb..da28d18 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
@@ -21,8 +21,8 @@
import java.util.List;
import java.util.Map;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.om.functions.ExternalFunctionInfo;
-import org.apache.asterix.om.functions.ExternalFunctionLanguage;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
diff --git a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtilTest.java b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtilTest.java
index 732ae09..17b1213 100644
--- a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtilTest.java
+++ b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtilTest.java
@@ -20,12 +20,12 @@
import java.util.LinkedList;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.transactions.TxnId;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.entities.Function;
-import org.apache.asterix.om.functions.ExternalFunctionLanguage;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
index 0cf42a6..bdaa3fe 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
@@ -21,6 +21,7 @@
import java.util.List;
import java.util.Map;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionInfo.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionInfo.java
index 8bd00fb..1db0c19 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionInfo.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionInfo.java
@@ -21,6 +21,7 @@
import java.util.List;
import java.util.Map;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
@@ -28,20 +29,20 @@
public interface IExternalFunctionInfo extends IFunctionInfo {
- public IResultTypeComputer getResultTypeComputer();
+ IResultTypeComputer getResultTypeComputer();
- public IAType getReturnType();
+ IAType getReturnType();
- public List<String> getExternalIdentifier();
+ List<String> getExternalIdentifier();
- public List<IAType> getArgumentList();
+ List<IAType> getArgumentList();
- public ExternalFunctionLanguage getLanguage();
+ ExternalFunctionLanguage getLanguage();
- public FunctionKind getKind();
+ FunctionKind getKind();
- public String getLibrary();
+ String getLibrary();
- public Map<String, String> getParams();
+ Map<String, String> getParams();
}
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 7cc4a3e..69ee90c 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -70,11 +70,13 @@
<test.includes>${global.test.includes}</test.includes>
<test.excludes>${global.test.excludes}</test.excludes>
<global.itest.includes>**/*IT.java,**/*IT.java,**/*ITCase.java</global.itest.includes>
- <global.itest.excludes/>
+ <global.itest.excludes>**/ExternalPythonFunctionIT.java</global.itest.excludes>
<itest.includes>${global.itest.includes}</itest.includes>
<itest.excludes>${global.itest.excludes}</itest.excludes>
<license.stage>compile</license.stage>
<resource.stage>process-classes</resource.stage>
+ <pyro-shim.stage>none</pyro-shim.stage>
+ <pytestlib.stage>none</pytestlib.stage>
<!-- Versions under dependencymanagement or used in many projects via properties -->
<algebricks.version>0.3.5-SNAPSHOT</algebricks.version>
@@ -287,14 +289,6 @@
<failOnWarning>true</failOnWarning>
<outputXML>true</outputXML>
</configuration>
- <executions>
- <execution>
- <phase>process-test-classes</phase>
- <goals>
- <goal>analyze-only</goal>
- </goals>
- </execution>
- </executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -586,6 +580,11 @@
<artifactId>impsort-maven-plugin</artifactId>
<version>1.2.0</version>
</plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.6.0</version>
+ </plugin>
</plugins>
</pluginManagement>
</build>
@@ -629,6 +628,19 @@
</properties>
</profile>
<profile>
+ <id>python-udfs</id>
+ <activation>
+ <file>
+ <exists>${python.path}</exists>
+ </file>
+ </activation>
+ <properties>
+ <pyro-shim.stage>process-classes</pyro-shim.stage>
+ <pytestlib.stage>generate-test-resources</pytestlib.stage>
+ <global.itest.excludes/>
+ </properties>
+ </profile>
+ <profile>
<id>invalid-tests</id>
<properties>
<invalid.tests />
@@ -1296,6 +1308,11 @@
<version>1.14</version>
</dependency>
<dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.4</version>
+ </dependency>
+ <dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
<version>8.3.0</version>
@@ -1360,6 +1377,21 @@
<artifactId>postgresql</artifactId>
<version>42.2.10</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ <version>4.5.11</version>
+ </dependency>
+ <dependency>
+ <groupId>net.razorvine</groupId>
+ <artifactId>pyrolite</artifactId>
+ <version>4.30</version>
+ </dependency>
+ <dependency>
+ <groupId>net.razorvine</groupId>
+ <artifactId>serpent</artifactId>
+ <version>1.23</version>
+ </dependency>
</dependencies>
</dependencyManagement>
diff --git a/asterixdb/src/main/licenses/content/raw.githubusercontent.com_irmen_Pyrolite_master_LICENSE.txt b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_irmen_Pyrolite_master_LICENSE.txt
new file mode 100644
index 0000000..ad923a6
--- /dev/null
+++ b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_irmen_Pyrolite_master_LICENSE.txt
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) by Irmen de Jong
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/base/IScalarEvaluator.java b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/base/IScalarEvaluator.java
index f5ef9c3..31f3d41 100644
--- a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/base/IScalarEvaluator.java
+++ b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/base/IScalarEvaluator.java
@@ -23,5 +23,5 @@
import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
public interface IScalarEvaluator {
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException;
+ void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/resources/IDeallocatable.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/resources/IDeallocatable.java
index 00f54d3..0d3bb1b 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/resources/IDeallocatable.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/resources/IDeallocatable.java
@@ -19,5 +19,5 @@
package org.apache.hyracks.api.resources;
public interface IDeallocatable {
- public void deallocate();
+ void deallocate();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
index e947d7a..510db51 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
@@ -90,7 +90,8 @@
TRUST_STORE_PATH(STRING, (String) null),
KEY_STORE_PASSWORD(STRING, (String) null),
IO_WORKERS_PER_PARTITION(POSITIVE_INTEGER, 2),
- IO_QUEUE_SIZE(POSITIVE_INTEGER, 10);
+ IO_QUEUE_SIZE(POSITIVE_INTEGER, 10),
+ PYTHON_HOME(STRING, "/usr/bin/python3");
private final IOptionType parser;
private final String defaultValueDescription;
@@ -223,6 +224,8 @@
return "Number of threads per partition used to write and read from storage";
case IO_QUEUE_SIZE:
return "Length of the queue used for requests to write and read";
+ case PYTHON_HOME:
+ return "Path to python interpreter";
default:
throw new IllegalStateException("Not yet implemented: " + this);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/deployment/DeploymentUtils.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/deployment/DeploymentUtils.java
index b900591..d07b648 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/deployment/DeploymentUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/deployment/DeploymentUtils.java
@@ -33,6 +33,7 @@
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
@@ -54,8 +55,13 @@
*/
public class DeploymentUtils {
- private static final String DEPLOYMENT = "applications";
+ public static final String DEPLOYMENT = "applications";
private static final Logger LOGGER = LogManager.getLogger();
+ private static final String SHIV_SUFFIX = ".pyz";
+ private static final String ZIP_SUFFIX = ".zip";
+ private static final String SHIV_ROOT = "SHIV_ROOT";
+ private static final String SHIV_ENTRY_POINT = "SHIV_ENTRY_POINT";
+ private static final String SHIV_INTERPRETER = "SHIV_INTERPRETER";
/**
* undeploy an existing deployment
@@ -209,17 +215,20 @@
HttpClient hc = HttpClientBuilder.create().build();
HttpGet get = new HttpGet(url.toString());
HttpResponse response = hc.execute(get);
- InputStream is = response.getEntity().getContent();
+ HttpEntity e = response.getEntity();
OutputStream os = new FileOutputStream(targetFile);
try {
- IOUtils.copyLarge(is, os);
+ e.writeTo(os);
} finally {
os.close();
- is.close();
}
}
if (extractFromArchive) {
- unzip(targetFile.getAbsolutePath(), deploymentDir);
+ if (targetFile.getAbsolutePath().endsWith(SHIV_SUFFIX)) {
+ shiv(targetFile.getAbsolutePath(), deploymentDir);
+ } else if (targetFile.getAbsolutePath().endsWith(ZIP_SUFFIX)) {
+ unzip(targetFile.getAbsolutePath(), deploymentDir);
+ }
}
downloadedFileURLs.add(targetFile.toURI().toURL());
}
@@ -258,4 +267,21 @@
}
}
}
+
+ public static void shiv(String sourceFile, String outputDir) throws IOException {
+ loadShim(outputDir, "pyro4.pyz");
+ unzip(sourceFile, outputDir);
+ unzip(outputDir + File.separator + "pyro4.pyz", outputDir);
+ loadShim(outputDir, "entrypoint.py");
+ }
+
+ public static void loadShim(String outputDir, String name) throws IOException {
+ try (InputStream is = DeploymentUtils.class.getClassLoader().getResourceAsStream(name)) {
+ if (is != null) {
+ IOUtils.copyLarge(is, new FileOutputStream(new File(outputDir + File.separator + name)));
+ } else {
+ throw new IOException("Classpath does not contain necessary Python resources!");
+ }
+ }
+ }
}