Enabled Feed Tests and Added External Library tests
Feed tests had been switched off for a while due to having too many
sporadic failures. Now, we are switching them back on.
In addition, a new set of tests have been added to test that external
library works as expected.
Change-Id: Idd1fccd136fa2645b2707bbf7c04e60991ae8d4a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/625
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
diff --git a/asterix-app/src/main/assembly/binary-assembly.xml b/asterix-app/src/main/assembly/binary-assembly.xml
index 013769f..91e2549 100644
--- a/asterix-app/src/main/assembly/binary-assembly.xml
+++ b/asterix-app/src/main/assembly/binary-assembly.xml
@@ -34,4 +34,14 @@
<outputDirectory>lib</outputDirectory>
</fileSet>
</fileSets>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>externallib</outputDirectory>
+ <includes>
+ <include>asterix-external-data:*:zip</include>
+ </includes>
+ <unpack>false</unpack>
+ <useTransitiveDependencies>false</useTransitiveDependencies>
+ </dependencySet>
+ </dependencySets>
</assembly>
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
index 4df461b..79ce721 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
@@ -29,7 +29,7 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.asterix.feed.CentralFeedManager;
+import org.apache.asterix.app.external.CentralFeedManager;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java
index 6957926..eacee6d 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java
@@ -32,13 +32,13 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import org.apache.asterix.app.external.CentralFeedManager;
import org.apache.asterix.external.feed.api.IFeedLoadManager;
import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
import org.apache.asterix.external.feed.management.FeedConnectionId;
import org.apache.asterix.external.feed.management.FeedId;
import org.apache.asterix.external.feed.watch.FeedActivity;
import org.apache.asterix.external.feed.watch.FeedActivity.FeedActivityDetails;
-import org.apache.asterix.feed.CentralFeedManager;
public class FeedServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java
index d459775..52a140d 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java
@@ -26,9 +26,9 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.asterix.app.external.FeedLifecycleListener;
import org.apache.asterix.external.feed.management.FeedConnectionId;
import org.apache.asterix.external.feed.message.RemoteSocketMessageListener;
-import org.apache.asterix.feed.FeedLifecycleListener;
public class FeedServletUtil {
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/CentralFeedManager.java b/asterix-app/src/main/java/org/apache/asterix/app/external/CentralFeedManager.java
similarity index 98%
rename from asterix-app/src/main/java/org/apache/asterix/feed/CentralFeedManager.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/CentralFeedManager.java
index 4020bde..cab5e64 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/CentralFeedManager.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/CentralFeedManager.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
import java.io.IOException;
import java.io.PrintWriter;
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java b/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
similarity index 99%
rename from asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
index 77c6a54..2f497f9 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.file;
+package org.apache.asterix.app.external;
import java.io.IOException;
import java.util.ArrayList;
@@ -52,6 +52,8 @@
import org.apache.asterix.external.operators.IndexInfoOperatorDescriptor;
import org.apache.asterix.external.provider.AdapterFactoryProvider;
import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.file.IndexOperations;
+import org.apache.asterix.file.JobSpecificationUtils;
import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import org.apache.asterix.formats.nontagged.AqlTypeTraitProvider;
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java b/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
similarity index 69%
rename from asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
index b0dfd58..a5654bd 100755
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.hyracks.bootstrap;
+package org.apache.asterix.app.external;
import java.io.File;
import java.io.FilenameFilter;
@@ -47,30 +47,39 @@
import org.apache.asterix.metadata.api.IMetadataEntity;
import org.apache.asterix.metadata.entities.DatasourceAdapter;
import org.apache.asterix.metadata.entities.Dataverse;
+import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.metadata.entities.Library;
import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
-public class ExternalLibraryBootstrap {
+public class ExternalLibraryUtils {
- private static Logger LOGGER = Logger.getLogger(ExternalLibraryBootstrap.class.getName());
+ private static Logger LOGGER = Logger.getLogger(ExternalLibraryUtils.class.getName());
public static void setUpExternaLibraries(boolean isMetadataNode) throws Exception {
+ // start by un-installing removed libraries (Metadata Node only)
Map<String, List<String>> uninstalledLibs = null;
if (isMetadataNode) {
uninstalledLibs = uninstallLibraries();
}
+ // get the directory of the to be installed libraries
File installLibDir = getLibraryInstallDir();
+ // directory exists?
if (installLibDir.exists()) {
+ // get the list of files in the directory
for (String dataverse : installLibDir.list()) {
File dataverseDir = new File(installLibDir, dataverse);
String[] libraries = dataverseDir.list();
for (String library : libraries) {
+ // for each file (library), register library
registerLibrary(dataverse, library, isMetadataNode, installLibDir);
+ // is metadata node?
if (isMetadataNode) {
+ // get library file
File libraryDir = new File(installLibDir.getAbsolutePath() + File.separator + dataverse
+ File.separator + library);
+ // install if needed (i,e, add the functions, adapters, datasources, parsers to the metadata) <Not required for use>
installLibraryIfNeeded(dataverse, libraryDir, uninstalledLibs);
}
}
@@ -78,18 +87,30 @@
}
}
+ /**
+ * un-install libraries.
+ * @return a map from dataverse -> list of uninstalled libraries.
+ * @throws Exception
+ */
private static Map<String, List<String>> uninstallLibraries() throws Exception {
Map<String, List<String>> uninstalledLibs = new HashMap<String, List<String>>();
+ // get the directory of the un-install libraries
File uninstallLibDir = getLibraryUninstallDir();
String[] uninstallLibNames;
+ // directory exists?
if (uninstallLibDir.exists()) {
+ // list files
uninstallLibNames = uninstallLibDir.list();
for (String uninstallLibName : uninstallLibNames) {
+ // Get the <dataverse name - library name> pair
String[] components = uninstallLibName.split("\\.");
String dataverse = components[0];
String libName = components[1];
+ // un-install
uninstallLibrary(dataverse, libName);
+ // delete the library file
new File(uninstallLibDir, uninstallLibName).delete();
+ // add the library to the list of uninstalled libraries
List<String> uinstalledLibsInDv = uninstalledLibs.get(dataverse);
if (uinstalledLibsInDv == null) {
uinstalledLibsInDv = new ArrayList<String>();
@@ -101,39 +122,56 @@
return uninstalledLibs;
}
- private static boolean uninstallLibrary(String dataverse, String libraryName)
+ /**
+ * Remove the library from metadata completely.
+ * TODO Currently, external libraries only include functions and adapters. we need to extend this to include:
+ * 1. external data source
+ * 2. data parser
+ * @param dataverse
+ * @param libraryName
+ * @return true if the library was found and removed, false otherwise
+ * @throws AsterixException
+ * @throws RemoteException
+ * @throws ACIDException
+ */
+ protected static boolean uninstallLibrary(String dataverse, String libraryName)
throws AsterixException, RemoteException, ACIDException {
MetadataTransactionContext mdTxnCtx = null;
try {
+ // begin transaction
mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ // make sure dataverse exists
Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse);
if (dv == null) {
return false;
}
-
- org.apache.asterix.metadata.entities.Library library = MetadataManager.INSTANCE.getLibrary(mdTxnCtx,
- dataverse, libraryName);
+ // make sure library exists
+ Library library = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName);
if (library == null) {
return false;
}
- List<org.apache.asterix.metadata.entities.Function> functions = MetadataManager.INSTANCE
- .getDataverseFunctions(mdTxnCtx, dataverse);
- for (org.apache.asterix.metadata.entities.Function function : functions) {
+ // get dataverse functions
+ List<Function> functions = MetadataManager.INSTANCE.getDataverseFunctions(mdTxnCtx, dataverse);
+ for (Function function : functions) {
+ // does function belong to library?
if (function.getName().startsWith(libraryName + "#")) {
+ // drop the function
MetadataManager.INSTANCE.dropFunction(mdTxnCtx,
new FunctionSignature(dataverse, function.getName(), function.getArity()));
}
}
- List<org.apache.asterix.metadata.entities.DatasourceAdapter> adapters = MetadataManager.INSTANCE
- .getDataverseAdapters(mdTxnCtx, dataverse);
- for (org.apache.asterix.metadata.entities.DatasourceAdapter adapter : adapters) {
+ // get the dataverse adapters
+ List<DatasourceAdapter> adapters = MetadataManager.INSTANCE.getDataverseAdapters(mdTxnCtx, dataverse);
+ for (DatasourceAdapter adapter : adapters) {
+ // belong to the library?
if (adapter.getAdapterIdentifier().getName().startsWith(libraryName + "#")) {
+ // remove adapter <! we didn't check if there are feeds which use this adapter>
MetadataManager.INSTANCE.dropAdapter(mdTxnCtx, dataverse, adapter.getAdapterIdentifier().getName());
}
}
-
+ // drop the library itself
MetadataManager.INSTANCE.dropLibrary(mdTxnCtx, dataverse, libraryName);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
} catch (Exception e) {
@@ -143,46 +181,59 @@
return true;
}
- // Each element of a library is installed as part of a transaction. Any
- // failure in installing an element does not effect installation of other
- // libraries
- private static void installLibraryIfNeeded(String dataverse, final File libraryDir,
+ /**
+ * Each element of a library is installed as part of a transaction. Any
+ * failure in installing an element does not effect installation of other
+ * libraries.
+ */
+ protected static void installLibraryIfNeeded(String dataverse, final File libraryDir,
Map<String, List<String>> uninstalledLibs) throws Exception {
String libraryName = libraryDir.getName().trim();
List<String> uninstalledLibsInDv = uninstalledLibs.get(dataverse);
+ // was this library just un-installed?
boolean wasUninstalled = uninstalledLibsInDv != null && uninstalledLibsInDv.contains(libraryName);
-
MetadataTransactionContext mdTxnCtx = null;
- MetadataManager.INSTANCE.acquireWriteLatch();
try {
mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- org.apache.asterix.metadata.entities.Library libraryInMetadata = MetadataManager.INSTANCE
- .getLibrary(mdTxnCtx, dataverse, libraryName);
+ Library libraryInMetadata = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName);
if (libraryInMetadata != null && !wasUninstalled) {
+ // exists in metadata and was not un-installed, we return.
+ // Another place which shows that our metadata transactions are broken (we didn't call commit before!!!)
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
return;
}
+ // Add library
+ MetadataManager.INSTANCE.addLibrary(mdTxnCtx, new Library(dataverse, libraryName));
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Added library " + libraryName + " to Metadata");
+ }
+
+ // Get the descriptor
String[] libraryDescriptors = libraryDir.list(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".xml");
}
});
-
ExternalLibrary library = getLibrary(new File(libraryDir + File.separator + libraryDescriptors[0]));
if (libraryDescriptors.length == 0) {
- throw new Exception("No library descriptor defined");
+ // should be fine. library was installed but its content was not added to metadata
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ return;
} else if (libraryDescriptors.length > 1) {
throw new Exception("More than 1 library descriptors defined");
}
+ // Get the dataverse
Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse);
if (dv == null) {
MetadataManager.INSTANCE.addDataverse(mdTxnCtx, new Dataverse(dataverse,
NonTaggedDataFormat.NON_TAGGED_DATA_FORMAT, IMetadataEntity.PENDING_NO_OP));
}
+ // Add functions
if (library.getLibraryFunctions() != null) {
for (LibraryFunction function : library.getLibraryFunctions().getLibraryFunction()) {
String[] fargs = function.getArguments().trim().split(",");
@@ -190,9 +241,8 @@
for (String arg : fargs) {
args.add(arg);
}
- org.apache.asterix.metadata.entities.Function f = new org.apache.asterix.metadata.entities.Function(
- dataverse, libraryName + "#" + function.getName().trim(), args.size(), args,
- function.getReturnType().trim(), function.getDefinition().trim(),
+ Function f = new Function(dataverse, libraryName + "#" + function.getName().trim(), args.size(),
+ args, function.getReturnType().trim(), function.getDefinition().trim(),
library.getLanguage().trim(), function.getFunctionType().trim());
MetadataManager.INSTANCE.addFunction(mdTxnCtx, f);
if (LOGGER.isLoggable(Level.INFO)) {
@@ -202,9 +252,10 @@
}
if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Installed functions contain in library :" + libraryName);
+ LOGGER.info("Installed functions in library :" + libraryName);
}
+ // Add adapters
if (library.getLibraryAdapters() != null) {
for (LibraryAdapter adapter : library.getLibraryAdapters().getLibraryAdapter()) {
String adapterFactoryClass = adapter.getFactoryClass().trim();
@@ -220,15 +271,8 @@
}
if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Installed adapters contain in library :" + libraryName);
+ LOGGER.info("Installed adapters in library :" + libraryName);
}
-
- MetadataManager.INSTANCE.addLibrary(mdTxnCtx, new Library(dataverse, libraryName));
-
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Added library " + libraryName + "to Metadata");
- }
-
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
} catch (Exception e) {
e.printStackTrace();
@@ -236,17 +280,31 @@
LOGGER.info("Exception in installing library " + libraryName);
}
MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
- } finally {
- MetadataManager.INSTANCE.releaseWriteLatch();
}
}
- private static void registerLibrary(String dataverse, String libraryName, boolean isMetadataNode,
+ /**
+ * register the library class loader with the external library manager
+ * @param dataverse
+ * @param libraryName
+ * @param isMetadataNode
+ * @param installLibDir
+ * @throws Exception
+ */
+ protected static void registerLibrary(String dataverse, String libraryName, boolean isMetadataNode,
File installLibDir) throws Exception {
+ // get the class loader
ClassLoader classLoader = getLibraryClassLoader(dataverse, libraryName);
+ // register it with the external library manager
ExternalLibraryManager.registerLibraryClassLoader(dataverse, libraryName, classLoader);
}
+ /**
+ * Get the library from the xml file
+ * @param libraryXMLPath
+ * @return
+ * @throws Exception
+ */
private static ExternalLibrary getLibrary(File libraryXMLPath) throws Exception {
JAXBContext configCtx = JAXBContext.newInstance(ExternalLibrary.class);
Unmarshaller unmarshaller = configCtx.createUnmarshaller();
@@ -254,14 +312,22 @@
return library;
}
+ /**
+ * Get the class loader for the library
+ * @param dataverse
+ * @param libraryName
+ * @return
+ * @throws Exception
+ */
private static ClassLoader getLibraryClassLoader(String dataverse, String libraryName) throws Exception {
-
+ // Get a reference to the library directory
File installDir = getLibraryInstallDir();
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Installing lirbary " + libraryName + " in dataverse " + dataverse + "."
+ " Install Directory: " + installDir.getAbsolutePath());
}
+ // get a reference to the specific library dir
File libDir = new File(
installDir.getAbsolutePath() + File.separator + dataverse + File.separator + libraryName);
FilenameFilter jarFileFilter = new FilenameFilter() {
@@ -271,6 +337,7 @@
}
};
+ // Get the jar file <Allow only a single jar file>
String[] jarsInLibDir = libDir.list(jarFileFilter);
if (jarsInLibDir.length > 1) {
throw new Exception("Incorrect library structure: found multiple library jars");
@@ -280,6 +347,7 @@
}
File libJar = new File(libDir, jarsInLibDir[0]);
+ // get the jar dependencies
File libDependencyDir = new File(libDir.getAbsolutePath() + File.separator + "lib");
int numDependencies = 1;
String[] libraryDependencies = null;
@@ -288,11 +356,13 @@
numDependencies += libraryDependencies.length;
}
- ClassLoader parentClassLoader = ExternalLibraryBootstrap.class.getClassLoader();
+ ClassLoader parentClassLoader = ExternalLibraryUtils.class.getClassLoader();
URL[] urls = new URL[numDependencies];
int count = 0;
+ // get url of library
urls[count++] = libJar.toURI().toURL();
+ // get urls for dependencies
if (libraryDependencies != null && libraryDependencies.length > 0) {
for (String dependency : libraryDependencies) {
File file = new File(libDependencyDir + File.separator + dependency);
@@ -308,16 +378,23 @@
LOGGER.info(logMesg.toString());
}
+ // create and return the class loader
ClassLoader classLoader = new URLClassLoader(urls, parentClassLoader);
return classLoader;
}
- private static File getLibraryInstallDir() {
+ /**
+ * @return the directory "$(pwd)/library": This needs to be improved
+ */
+ protected static File getLibraryInstallDir() {
String workingDir = System.getProperty("user.dir");
return new File(workingDir + File.separator + "library");
}
- private static File getLibraryUninstallDir() {
+ /**
+ * @return the directory "$(pwd)/uninstall": This needs to be improved
+ */
+ protected static File getLibraryUninstallDir() {
String workingDir = System.getProperty("user.dir");
return new File(workingDir + File.separator + "uninstall");
}
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedJobNotificationHandler.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedJobNotificationHandler.java
similarity index 99%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedJobNotificationHandler.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedJobNotificationHandler.java
index 49b88ca..d729680 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedJobNotificationHandler.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedJobNotificationHandler.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
import java.rmi.RemoteException;
import java.util.ArrayList;
@@ -33,6 +33,8 @@
import java.util.logging.Logger;
import org.apache.commons.lang3.StringUtils;
+import org.apache.asterix.app.external.FeedLifecycleListener.Message;
+import org.apache.asterix.app.external.FeedWorkCollection.SubscribeFeedWork;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.external.feed.api.IFeedJoint;
import org.apache.asterix.external.feed.api.IFeedLifecycleEventSubscriber;
@@ -55,8 +57,6 @@
import org.apache.asterix.external.operators.FeedCollectOperatorDescriptor;
import org.apache.asterix.external.operators.FeedIntakeOperatorDescriptor;
import org.apache.asterix.external.operators.FeedMetaOperatorDescriptor;
-import org.apache.asterix.feed.FeedLifecycleListener.Message;
-import org.apache.asterix.feed.FeedWorkCollection.SubscribeFeedWork;
import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies;
import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.hyracks.algebricks.common.utils.Pair;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedJoint.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedJoint.java
similarity index 99%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedJoint.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedJoint.java
index 43f227d..e650a5b 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedJoint.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedJoint.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
import java.util.ArrayList;
import java.util.List;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedLifecycleListener.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedLifecycleListener.java
similarity index 99%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedLifecycleListener.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedLifecycleListener.java
index aac3675..8e44af4 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedLifecycleListener.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedLifecycleListener.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
import java.io.PrintWriter;
import java.util.ArrayList;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedLoadManager.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedLoadManager.java
similarity index 99%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedLoadManager.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedLoadManager.java
index 18e885d..5a590b4 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedLoadManager.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedLoadManager.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
import java.util.ArrayList;
import java.util.Collection;
@@ -45,7 +45,6 @@
import org.apache.asterix.external.feed.watch.FeedActivity;
import org.apache.asterix.external.feed.watch.NodeLoadReport;
import org.apache.asterix.external.feed.watch.FeedJobInfo.FeedJobState;
-import org.apache.asterix.file.FeedOperations;
import org.apache.asterix.metadata.feeds.FeedMetadataUtil;
import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.hyracks.algebricks.common.utils.Pair;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedMessageReceiver.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedMessageReceiver.java
similarity index 97%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedMessageReceiver.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedMessageReceiver.java
index 4ae2e59..bff1a4d 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedMessageReceiver.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedMessageReceiver.java
@@ -16,13 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
import java.util.logging.Level;
+import org.apache.asterix.app.external.CentralFeedManager.AQLExecutor;
import org.apache.asterix.external.feed.api.IFeedLoadManager;
-import org.apache.asterix.external.feed.api.IFeedTrackingManager;
import org.apache.asterix.external.feed.api.IFeedMessage.MessageType;
+import org.apache.asterix.external.feed.api.IFeedTrackingManager;
import org.apache.asterix.external.feed.message.FeedCongestionMessage;
import org.apache.asterix.external.feed.message.FeedReportMessage;
import org.apache.asterix.external.feed.message.FeedTupleCommitAckMessage;
@@ -32,7 +33,6 @@
import org.apache.asterix.external.feed.message.ThrottlingEnabledFeedMessage;
import org.apache.asterix.external.feed.watch.NodeLoadReport;
import org.apache.asterix.external.util.FeedConstants;
-import org.apache.asterix.feed.CentralFeedManager.AQLExecutor;
import org.apache.asterix.hyracks.bootstrap.FeedBootstrap;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.json.JSONObject;
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedOperations.java
similarity index 99%
rename from asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedOperations.java
index 6a036c0..c0245d7 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedOperations.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.file;
+package org.apache.asterix.app.external;
import java.util.Collection;
import java.util.List;
@@ -37,7 +37,7 @@
import org.apache.asterix.external.feed.watch.FeedConnectJobInfo;
import org.apache.asterix.external.operators.FeedMessageOperatorDescriptor;
import org.apache.asterix.external.util.FeedConstants;
-import org.apache.asterix.feed.FeedLifecycleListener;
+import org.apache.asterix.file.JobSpecificationUtils;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.entities.Feed;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedTrackingManager.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedTrackingManager.java
similarity index 98%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedTrackingManager.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedTrackingManager.java
index a1c6fb9c..29230c1 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedTrackingManager.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedTrackingManager.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
import java.util.Arrays;
import java.util.BitSet;
@@ -32,7 +32,6 @@
import org.apache.asterix.external.feed.management.FeedConnectionId;
import org.apache.asterix.external.feed.message.FeedTupleCommitAckMessage;
import org.apache.asterix.external.feed.message.FeedTupleCommitResponseMessage;
-import org.apache.asterix.file.FeedOperations;
import org.apache.hyracks.api.job.JobSpecification;
public class FeedTrackingManager implements IFeedTrackingManager {
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkCollection.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java
similarity index 99%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkCollection.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java
index 9d746c8..53b9792 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkCollection.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
import java.io.PrintWriter;
import java.util.ArrayList;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkRequestResponseHandler.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkRequestResponseHandler.java
similarity index 99%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkRequestResponseHandler.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkRequestResponseHandler.java
index b30d8a7..2dc1162 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkRequestResponseHandler.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkRequestResponseHandler.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
import java.util.ArrayList;
import java.util.HashMap;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedsActivator.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedsActivator.java
similarity index 98%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedsActivator.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedsActivator.java
index dc02a53..5a6d28e 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedsActivator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedsActivator.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
import java.io.PrintWriter;
import java.util.ArrayList;
diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 46ea72b..9f024e9 100644
--- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -42,6 +42,11 @@
import org.apache.asterix.api.common.APIFramework;
import org.apache.asterix.api.common.SessionConfig;
import org.apache.asterix.api.common.SessionConfig.OutputFormat;
+import org.apache.asterix.app.external.CentralFeedManager;
+import org.apache.asterix.app.external.ExternalIndexingOperations;
+import org.apache.asterix.app.external.FeedJoint;
+import org.apache.asterix.app.external.FeedLifecycleListener;
+import org.apache.asterix.app.external.FeedOperations;
import org.apache.asterix.common.config.AsterixExternalProperties;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
@@ -69,13 +74,8 @@
import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
import org.apache.asterix.external.feed.watch.FeedActivity.FeedActivityDetails;
import org.apache.asterix.external.indexing.ExternalFile;
-import org.apache.asterix.feed.CentralFeedManager;
-import org.apache.asterix.feed.FeedJoint;
-import org.apache.asterix.feed.FeedLifecycleListener;
import org.apache.asterix.file.DatasetOperations;
import org.apache.asterix.file.DataverseOperations;
-import org.apache.asterix.file.ExternalIndexingOperations;
-import org.apache.asterix.file.FeedOperations;
import org.apache.asterix.file.IndexOperations;
import org.apache.asterix.formats.nontagged.AqlTypeTraitProvider;
import org.apache.asterix.lang.aql.statement.SubscribeFeedStatement;
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
index 54cac09..24df771 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
@@ -24,6 +24,7 @@
import java.util.List;
import java.util.Map;
+import org.apache.asterix.app.external.ExternalIndexingOperations;
import org.apache.asterix.common.config.AsterixStorageProperties;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
index 83f91a0..be1c356 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
@@ -20,6 +20,7 @@
import java.util.List;
+import org.apache.asterix.app.external.ExternalIndexingOperations;
import org.apache.asterix.common.api.ILocalResourceMetadata;
import org.apache.asterix.common.config.AsterixStorageProperties;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
index adf0a4d..e683ef4 100644
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
@@ -34,6 +34,9 @@
import org.apache.asterix.api.http.servlet.ShutdownAPIServlet;
import org.apache.asterix.api.http.servlet.UpdateAPIServlet;
import org.apache.asterix.api.http.servlet.VersionAPIServlet;
+import org.apache.asterix.app.external.CentralFeedManager;
+import org.apache.asterix.app.external.ExternalLibraryUtils;
+import org.apache.asterix.app.external.FeedLifecycleListener;
import org.apache.asterix.common.api.AsterixThreadFactory;
import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
import org.apache.asterix.common.config.AsterixExternalProperties;
@@ -43,8 +46,6 @@
import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
import org.apache.asterix.event.service.ILookupService;
import org.apache.asterix.external.feed.api.ICentralFeedManager;
-import org.apache.asterix.feed.CentralFeedManager;
-import org.apache.asterix.feed.FeedLifecycleListener;
import org.apache.asterix.messaging.CCMessageBroker;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.api.IAsterixStateProxy;
@@ -110,7 +111,7 @@
setupFeedServer(externalProperties);
feedServer.start();
- ExternalLibraryBootstrap.setUpExternaLibraries(false);
+ ExternalLibraryUtils.setUpExternaLibraries(false);
centralFeedManager = CentralFeedManager.getInstance();
centralFeedManager.start();
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/FeedBootstrap.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/FeedBootstrap.java
index d5f1a51..a6be075 100644
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/FeedBootstrap.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/FeedBootstrap.java
@@ -18,9 +18,9 @@
*/
package org.apache.asterix.hyracks.bootstrap;
+import org.apache.asterix.app.external.CentralFeedManager;
import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.external.util.FeedConstants;
-import org.apache.asterix.feed.CentralFeedManager;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
index 2bac1cf..8132d4b 100644
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
@@ -23,6 +23,8 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.asterix.app.external.CentralFeedManager;
+import org.apache.asterix.app.external.ExternalIndexingOperations;
import org.apache.asterix.common.api.IClusterManagementWork;
import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
import org.apache.asterix.common.api.IClusterManagementWorkResponse;
@@ -32,8 +34,6 @@
import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.external.indexing.ExternalFile;
-import org.apache.asterix.feed.CentralFeedManager;
-import org.apache.asterix.file.ExternalIndexingOperations;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index fcb196d..4922ae6 100644
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -26,6 +26,7 @@
import java.util.logging.Logger;
import org.apache.asterix.api.common.AsterixAppRuntimeContext;
+import org.apache.asterix.app.external.ExternalLibraryUtils;
import org.apache.asterix.common.api.AsterixThreadFactory;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.common.config.AsterixMetadataProperties;
@@ -212,7 +213,7 @@
if (isMetadataNode && !pendingFailbackCompletion) {
runtimeContext.initializeMetadata(systemState == SystemState.NEW_UNIVERSE);
}
- ExternalLibraryBootstrap.setUpExternaLibraries(isMetadataNode && !pendingFailbackCompletion);
+ ExternalLibraryUtils.setUpExternaLibraries(isMetadataNode && !pendingFailbackCompletion);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Starting lifecycle components");