Enabled Feed Tests and Added External Library tests

Feed tests had been switched off for a while due to having too many
sporadic failures. Now, we are switching them back on.
In addition, a new set of tests have been added to test that external
library works as expected.

Change-Id: Idd1fccd136fa2645b2707bbf7c04e60991ae8d4a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/625
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
diff --git a/asterix-app/src/main/assembly/binary-assembly.xml b/asterix-app/src/main/assembly/binary-assembly.xml
index 013769f..91e2549 100644
--- a/asterix-app/src/main/assembly/binary-assembly.xml
+++ b/asterix-app/src/main/assembly/binary-assembly.xml
@@ -34,4 +34,14 @@
 			<outputDirectory>lib</outputDirectory>
 		</fileSet>
 	</fileSets>
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>externallib</outputDirectory>
+            <includes>
+                <include>asterix-external-data:*:zip</include>
+            </includes>
+            <unpack>false</unpack>
+            <useTransitiveDependencies>false</useTransitiveDependencies>
+        </dependencySet>
+    </dependencySets>
 </assembly>
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
index 4df461b..79ce721 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
@@ -29,7 +29,7 @@
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import org.apache.asterix.feed.CentralFeedManager;
+import org.apache.asterix.app.external.CentralFeedManager;
 import org.apache.asterix.metadata.MetadataManager;
 import org.apache.asterix.metadata.MetadataTransactionContext;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java
index 6957926..eacee6d 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java
@@ -32,13 +32,13 @@
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import org.apache.asterix.app.external.CentralFeedManager;
 import org.apache.asterix.external.feed.api.IFeedLoadManager;
 import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
 import org.apache.asterix.external.feed.management.FeedConnectionId;
 import org.apache.asterix.external.feed.management.FeedId;
 import org.apache.asterix.external.feed.watch.FeedActivity;
 import org.apache.asterix.external.feed.watch.FeedActivity.FeedActivityDetails;
-import org.apache.asterix.feed.CentralFeedManager;
 
 public class FeedServlet extends HttpServlet {
     private static final long serialVersionUID = 1L;
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java
index d459775..52a140d 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java
@@ -26,9 +26,9 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import org.apache.asterix.app.external.FeedLifecycleListener;
 import org.apache.asterix.external.feed.management.FeedConnectionId;
 import org.apache.asterix.external.feed.message.RemoteSocketMessageListener;
-import org.apache.asterix.feed.FeedLifecycleListener;
 
 public class FeedServletUtil {
 
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/CentralFeedManager.java b/asterix-app/src/main/java/org/apache/asterix/app/external/CentralFeedManager.java
similarity index 98%
rename from asterix-app/src/main/java/org/apache/asterix/feed/CentralFeedManager.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/CentralFeedManager.java
index 4020bde..cab5e64 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/CentralFeedManager.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/CentralFeedManager.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
 
 import java.io.IOException;
 import java.io.PrintWriter;
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java b/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
similarity index 99%
rename from asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
index 77c6a54..2f497f9 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.file;
+package org.apache.asterix.app.external;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -52,6 +52,8 @@
 import org.apache.asterix.external.operators.IndexInfoOperatorDescriptor;
 import org.apache.asterix.external.provider.AdapterFactoryProvider;
 import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.file.IndexOperations;
+import org.apache.asterix.file.JobSpecificationUtils;
 import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
 import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
 import org.apache.asterix.formats.nontagged.AqlTypeTraitProvider;
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java b/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
similarity index 69%
rename from asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
index b0dfd58..a5654bd 100755
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.hyracks.bootstrap;
+package org.apache.asterix.app.external;
 
 import java.io.File;
 import java.io.FilenameFilter;
@@ -47,30 +47,39 @@
 import org.apache.asterix.metadata.api.IMetadataEntity;
 import org.apache.asterix.metadata.entities.DatasourceAdapter;
 import org.apache.asterix.metadata.entities.Dataverse;
+import org.apache.asterix.metadata.entities.Function;
 import org.apache.asterix.metadata.entities.Library;
 import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
 
-public class ExternalLibraryBootstrap {
+public class ExternalLibraryUtils {
 
-    private static Logger LOGGER = Logger.getLogger(ExternalLibraryBootstrap.class.getName());
+    private static Logger LOGGER = Logger.getLogger(ExternalLibraryUtils.class.getName());
 
     public static void setUpExternaLibraries(boolean isMetadataNode) throws Exception {
 
+        // start by un-installing removed libraries (Metadata Node only)
         Map<String, List<String>> uninstalledLibs = null;
         if (isMetadataNode) {
             uninstalledLibs = uninstallLibraries();
         }
 
+        // get the directory of the to be installed libraries
         File installLibDir = getLibraryInstallDir();
+        // directory exists?
         if (installLibDir.exists()) {
+            // get the list of files in the directory
             for (String dataverse : installLibDir.list()) {
                 File dataverseDir = new File(installLibDir, dataverse);
                 String[] libraries = dataverseDir.list();
                 for (String library : libraries) {
+                    // for each file (library), register library
                     registerLibrary(dataverse, library, isMetadataNode, installLibDir);
+                    // is metadata node?
                     if (isMetadataNode) {
+                        // get library file
                         File libraryDir = new File(installLibDir.getAbsolutePath() + File.separator + dataverse
                                 + File.separator + library);
+                        // install if needed (i,e, add the functions, adapters, datasources, parsers to the metadata) <Not required for use>
                         installLibraryIfNeeded(dataverse, libraryDir, uninstalledLibs);
                     }
                 }
@@ -78,18 +87,30 @@
         }
     }
 
+    /**
+     * un-install libraries.
+     * @return a map from dataverse -> list of uninstalled libraries.
+     * @throws Exception
+     */
     private static Map<String, List<String>> uninstallLibraries() throws Exception {
         Map<String, List<String>> uninstalledLibs = new HashMap<String, List<String>>();
+        // get the directory of the un-install libraries
         File uninstallLibDir = getLibraryUninstallDir();
         String[] uninstallLibNames;
+        // directory exists?
         if (uninstallLibDir.exists()) {
+            // list files
             uninstallLibNames = uninstallLibDir.list();
             for (String uninstallLibName : uninstallLibNames) {
+                // Get the <dataverse name - library name> pair
                 String[] components = uninstallLibName.split("\\.");
                 String dataverse = components[0];
                 String libName = components[1];
+                // un-install
                 uninstallLibrary(dataverse, libName);
+                // delete the library file
                 new File(uninstallLibDir, uninstallLibName).delete();
+                // add the library to the list of uninstalled libraries
                 List<String> uinstalledLibsInDv = uninstalledLibs.get(dataverse);
                 if (uinstalledLibsInDv == null) {
                     uinstalledLibsInDv = new ArrayList<String>();
@@ -101,39 +122,56 @@
         return uninstalledLibs;
     }
 
-    private static boolean uninstallLibrary(String dataverse, String libraryName)
+    /**
+     * Remove the library from metadata completely.
+     * TODO Currently, external libraries only include functions and adapters. we need to extend this to include:
+     * 1. external data source
+     * 2. data parser
+     * @param dataverse
+     * @param libraryName
+     * @return true if the library was found and removed, false otherwise
+     * @throws AsterixException
+     * @throws RemoteException
+     * @throws ACIDException
+     */
+    protected static boolean uninstallLibrary(String dataverse, String libraryName)
             throws AsterixException, RemoteException, ACIDException {
         MetadataTransactionContext mdTxnCtx = null;
         try {
+            // begin transaction
             mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+            // make sure dataverse exists
             Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse);
             if (dv == null) {
                 return false;
             }
-
-            org.apache.asterix.metadata.entities.Library library = MetadataManager.INSTANCE.getLibrary(mdTxnCtx,
-                    dataverse, libraryName);
+            // make sure library exists
+            Library library = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName);
             if (library == null) {
                 return false;
             }
 
-            List<org.apache.asterix.metadata.entities.Function> functions = MetadataManager.INSTANCE
-                    .getDataverseFunctions(mdTxnCtx, dataverse);
-            for (org.apache.asterix.metadata.entities.Function function : functions) {
+            // get dataverse functions
+            List<Function> functions = MetadataManager.INSTANCE.getDataverseFunctions(mdTxnCtx, dataverse);
+            for (Function function : functions) {
+                // does function belong to library?
                 if (function.getName().startsWith(libraryName + "#")) {
+                    // drop the function
                     MetadataManager.INSTANCE.dropFunction(mdTxnCtx,
                             new FunctionSignature(dataverse, function.getName(), function.getArity()));
                 }
             }
 
-            List<org.apache.asterix.metadata.entities.DatasourceAdapter> adapters = MetadataManager.INSTANCE
-                    .getDataverseAdapters(mdTxnCtx, dataverse);
-            for (org.apache.asterix.metadata.entities.DatasourceAdapter adapter : adapters) {
+            // get the dataverse adapters
+            List<DatasourceAdapter> adapters = MetadataManager.INSTANCE.getDataverseAdapters(mdTxnCtx, dataverse);
+            for (DatasourceAdapter adapter : adapters) {
+                // belong to the library?
                 if (adapter.getAdapterIdentifier().getName().startsWith(libraryName + "#")) {
+                    // remove adapter <! we didn't check if there are feeds which use this adapter>
                     MetadataManager.INSTANCE.dropAdapter(mdTxnCtx, dataverse, adapter.getAdapterIdentifier().getName());
                 }
             }
-
+            // drop the library itself
             MetadataManager.INSTANCE.dropLibrary(mdTxnCtx, dataverse, libraryName);
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
         } catch (Exception e) {
@@ -143,46 +181,59 @@
         return true;
     }
 
-    // Each element of a library is installed as part of a transaction. Any
-    // failure in installing an element does not effect installation of other
-    // libraries
-    private static void installLibraryIfNeeded(String dataverse, final File libraryDir,
+    /**
+     *  Each element of a library is installed as part of a transaction. Any
+     *  failure in installing an element does not effect installation of other
+     *  libraries.
+     */
+    protected static void installLibraryIfNeeded(String dataverse, final File libraryDir,
             Map<String, List<String>> uninstalledLibs) throws Exception {
 
         String libraryName = libraryDir.getName().trim();
         List<String> uninstalledLibsInDv = uninstalledLibs.get(dataverse);
+        // was this library just un-installed?
         boolean wasUninstalled = uninstalledLibsInDv != null && uninstalledLibsInDv.contains(libraryName);
-
         MetadataTransactionContext mdTxnCtx = null;
-        MetadataManager.INSTANCE.acquireWriteLatch();
         try {
             mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
-            org.apache.asterix.metadata.entities.Library libraryInMetadata = MetadataManager.INSTANCE
-                    .getLibrary(mdTxnCtx, dataverse, libraryName);
+            Library libraryInMetadata = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName);
             if (libraryInMetadata != null && !wasUninstalled) {
+                // exists in metadata and was not un-installed, we return.
+                // Another place which shows that our metadata transactions are broken (we didn't call commit before!!!)
+                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
                 return;
             }
 
+            // Add library
+            MetadataManager.INSTANCE.addLibrary(mdTxnCtx, new Library(dataverse, libraryName));
+            if (LOGGER.isLoggable(Level.INFO)) {
+                LOGGER.info("Added library " + libraryName + " to Metadata");
+            }
+
+            // Get the descriptor
             String[] libraryDescriptors = libraryDir.list(new FilenameFilter() {
                 @Override
                 public boolean accept(File dir, String name) {
                     return name.endsWith(".xml");
                 }
             });
-
             ExternalLibrary library = getLibrary(new File(libraryDir + File.separator + libraryDescriptors[0]));
 
             if (libraryDescriptors.length == 0) {
-                throw new Exception("No library descriptor defined");
+                // should be fine. library was installed but its content was not added to metadata
+                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                return;
             } else if (libraryDescriptors.length > 1) {
                 throw new Exception("More than 1 library descriptors defined");
             }
 
+            // Get the dataverse
             Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse);
             if (dv == null) {
                 MetadataManager.INSTANCE.addDataverse(mdTxnCtx, new Dataverse(dataverse,
                         NonTaggedDataFormat.NON_TAGGED_DATA_FORMAT, IMetadataEntity.PENDING_NO_OP));
             }
+            // Add functions
             if (library.getLibraryFunctions() != null) {
                 for (LibraryFunction function : library.getLibraryFunctions().getLibraryFunction()) {
                     String[] fargs = function.getArguments().trim().split(",");
@@ -190,9 +241,8 @@
                     for (String arg : fargs) {
                         args.add(arg);
                     }
-                    org.apache.asterix.metadata.entities.Function f = new org.apache.asterix.metadata.entities.Function(
-                            dataverse, libraryName + "#" + function.getName().trim(), args.size(), args,
-                            function.getReturnType().trim(), function.getDefinition().trim(),
+                    Function f = new Function(dataverse, libraryName + "#" + function.getName().trim(), args.size(),
+                            args, function.getReturnType().trim(), function.getDefinition().trim(),
                             library.getLanguage().trim(), function.getFunctionType().trim());
                     MetadataManager.INSTANCE.addFunction(mdTxnCtx, f);
                     if (LOGGER.isLoggable(Level.INFO)) {
@@ -202,9 +252,10 @@
             }
 
             if (LOGGER.isLoggable(Level.INFO)) {
-                LOGGER.info("Installed functions contain in library :" + libraryName);
+                LOGGER.info("Installed functions in library :" + libraryName);
             }
 
+            // Add adapters
             if (library.getLibraryAdapters() != null) {
                 for (LibraryAdapter adapter : library.getLibraryAdapters().getLibraryAdapter()) {
                     String adapterFactoryClass = adapter.getFactoryClass().trim();
@@ -220,15 +271,8 @@
             }
 
             if (LOGGER.isLoggable(Level.INFO)) {
-                LOGGER.info("Installed adapters contain in library :" + libraryName);
+                LOGGER.info("Installed adapters in library :" + libraryName);
             }
-
-            MetadataManager.INSTANCE.addLibrary(mdTxnCtx, new Library(dataverse, libraryName));
-
-            if (LOGGER.isLoggable(Level.INFO)) {
-                LOGGER.info("Added library " + libraryName + "to Metadata");
-            }
-
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
         } catch (Exception e) {
             e.printStackTrace();
@@ -236,17 +280,31 @@
                 LOGGER.info("Exception in installing library " + libraryName);
             }
             MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
-        } finally {
-            MetadataManager.INSTANCE.releaseWriteLatch();
         }
     }
 
-    private static void registerLibrary(String dataverse, String libraryName, boolean isMetadataNode,
+    /**
+     * register the library class loader with the external library manager
+     * @param dataverse
+     * @param libraryName
+     * @param isMetadataNode
+     * @param installLibDir
+     * @throws Exception
+     */
+    protected static void registerLibrary(String dataverse, String libraryName, boolean isMetadataNode,
             File installLibDir) throws Exception {
+        // get the class loader
         ClassLoader classLoader = getLibraryClassLoader(dataverse, libraryName);
+        // register it with the external library manager
         ExternalLibraryManager.registerLibraryClassLoader(dataverse, libraryName, classLoader);
     }
 
+    /**
+     * Get the library from the xml file
+     * @param libraryXMLPath
+     * @return
+     * @throws Exception
+     */
     private static ExternalLibrary getLibrary(File libraryXMLPath) throws Exception {
         JAXBContext configCtx = JAXBContext.newInstance(ExternalLibrary.class);
         Unmarshaller unmarshaller = configCtx.createUnmarshaller();
@@ -254,14 +312,22 @@
         return library;
     }
 
+    /**
+     * Get the class loader for the library
+     * @param dataverse
+     * @param libraryName
+     * @return
+     * @throws Exception
+     */
     private static ClassLoader getLibraryClassLoader(String dataverse, String libraryName) throws Exception {
-
+        // Get a reference to the library directory
         File installDir = getLibraryInstallDir();
         if (LOGGER.isLoggable(Level.INFO)) {
             LOGGER.info("Installing lirbary " + libraryName + " in dataverse " + dataverse + "."
                     + " Install Directory: " + installDir.getAbsolutePath());
         }
 
+        // get a reference to the specific library dir
         File libDir = new File(
                 installDir.getAbsolutePath() + File.separator + dataverse + File.separator + libraryName);
         FilenameFilter jarFileFilter = new FilenameFilter() {
@@ -271,6 +337,7 @@
             }
         };
 
+        // Get the jar file <Allow only a single jar file>
         String[] jarsInLibDir = libDir.list(jarFileFilter);
         if (jarsInLibDir.length > 1) {
             throw new Exception("Incorrect library structure: found multiple library jars");
@@ -280,6 +347,7 @@
         }
 
         File libJar = new File(libDir, jarsInLibDir[0]);
+        // get the jar dependencies
         File libDependencyDir = new File(libDir.getAbsolutePath() + File.separator + "lib");
         int numDependencies = 1;
         String[] libraryDependencies = null;
@@ -288,11 +356,13 @@
             numDependencies += libraryDependencies.length;
         }
 
-        ClassLoader parentClassLoader = ExternalLibraryBootstrap.class.getClassLoader();
+        ClassLoader parentClassLoader = ExternalLibraryUtils.class.getClassLoader();
         URL[] urls = new URL[numDependencies];
         int count = 0;
+        // get url of library
         urls[count++] = libJar.toURI().toURL();
 
+        // get urls for dependencies
         if (libraryDependencies != null && libraryDependencies.length > 0) {
             for (String dependency : libraryDependencies) {
                 File file = new File(libDependencyDir + File.separator + dependency);
@@ -308,16 +378,23 @@
             LOGGER.info(logMesg.toString());
         }
 
+        // create and return the class loader
         ClassLoader classLoader = new URLClassLoader(urls, parentClassLoader);
         return classLoader;
     }
 
-    private static File getLibraryInstallDir() {
+    /**
+     *  @return the directory "$(pwd)/library": This needs to be improved
+     */
+    protected static File getLibraryInstallDir() {
         String workingDir = System.getProperty("user.dir");
         return new File(workingDir + File.separator + "library");
     }
 
-    private static File getLibraryUninstallDir() {
+    /**
+     * @return the directory "$(pwd)/uninstall": This needs to be improved
+     */
+    protected static File getLibraryUninstallDir() {
         String workingDir = System.getProperty("user.dir");
         return new File(workingDir + File.separator + "uninstall");
     }
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedJobNotificationHandler.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedJobNotificationHandler.java
similarity index 99%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedJobNotificationHandler.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedJobNotificationHandler.java
index 49b88ca..d729680 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedJobNotificationHandler.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedJobNotificationHandler.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
 
 import java.rmi.RemoteException;
 import java.util.ArrayList;
@@ -33,6 +33,8 @@
 import java.util.logging.Logger;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.asterix.app.external.FeedLifecycleListener.Message;
+import org.apache.asterix.app.external.FeedWorkCollection.SubscribeFeedWork;
 import org.apache.asterix.common.exceptions.ACIDException;
 import org.apache.asterix.external.feed.api.IFeedJoint;
 import org.apache.asterix.external.feed.api.IFeedLifecycleEventSubscriber;
@@ -55,8 +57,6 @@
 import org.apache.asterix.external.operators.FeedCollectOperatorDescriptor;
 import org.apache.asterix.external.operators.FeedIntakeOperatorDescriptor;
 import org.apache.asterix.external.operators.FeedMetaOperatorDescriptor;
-import org.apache.asterix.feed.FeedLifecycleListener.Message;
-import org.apache.asterix.feed.FeedWorkCollection.SubscribeFeedWork;
 import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies;
 import org.apache.asterix.om.util.AsterixAppContextInfo;
 import org.apache.hyracks.algebricks.common.utils.Pair;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedJoint.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedJoint.java
similarity index 99%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedJoint.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedJoint.java
index 43f227d..e650a5b 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedJoint.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedJoint.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
 
 import java.util.ArrayList;
 import java.util.List;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedLifecycleListener.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedLifecycleListener.java
similarity index 99%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedLifecycleListener.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedLifecycleListener.java
index aac3675..8e44af4 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedLifecycleListener.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedLifecycleListener.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
 
 import java.io.PrintWriter;
 import java.util.ArrayList;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedLoadManager.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedLoadManager.java
similarity index 99%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedLoadManager.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedLoadManager.java
index 18e885d..5a590b4 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedLoadManager.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedLoadManager.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -45,7 +45,6 @@
 import org.apache.asterix.external.feed.watch.FeedActivity;
 import org.apache.asterix.external.feed.watch.NodeLoadReport;
 import org.apache.asterix.external.feed.watch.FeedJobInfo.FeedJobState;
-import org.apache.asterix.file.FeedOperations;
 import org.apache.asterix.metadata.feeds.FeedMetadataUtil;
 import org.apache.asterix.om.util.AsterixAppContextInfo;
 import org.apache.hyracks.algebricks.common.utils.Pair;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedMessageReceiver.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedMessageReceiver.java
similarity index 97%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedMessageReceiver.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedMessageReceiver.java
index 4ae2e59..bff1a4d 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedMessageReceiver.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedMessageReceiver.java
@@ -16,13 +16,14 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
 
 import java.util.logging.Level;
 
+import org.apache.asterix.app.external.CentralFeedManager.AQLExecutor;
 import org.apache.asterix.external.feed.api.IFeedLoadManager;
-import org.apache.asterix.external.feed.api.IFeedTrackingManager;
 import org.apache.asterix.external.feed.api.IFeedMessage.MessageType;
+import org.apache.asterix.external.feed.api.IFeedTrackingManager;
 import org.apache.asterix.external.feed.message.FeedCongestionMessage;
 import org.apache.asterix.external.feed.message.FeedReportMessage;
 import org.apache.asterix.external.feed.message.FeedTupleCommitAckMessage;
@@ -32,7 +33,6 @@
 import org.apache.asterix.external.feed.message.ThrottlingEnabledFeedMessage;
 import org.apache.asterix.external.feed.watch.NodeLoadReport;
 import org.apache.asterix.external.util.FeedConstants;
-import org.apache.asterix.feed.CentralFeedManager.AQLExecutor;
 import org.apache.asterix.hyracks.bootstrap.FeedBootstrap;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.json.JSONObject;
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedOperations.java
similarity index 99%
rename from asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedOperations.java
index 6a036c0..c0245d7 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedOperations.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.file;
+package org.apache.asterix.app.external;
 
 import java.util.Collection;
 import java.util.List;
@@ -37,7 +37,7 @@
 import org.apache.asterix.external.feed.watch.FeedConnectJobInfo;
 import org.apache.asterix.external.operators.FeedMessageOperatorDescriptor;
 import org.apache.asterix.external.util.FeedConstants;
-import org.apache.asterix.feed.FeedLifecycleListener;
+import org.apache.asterix.file.JobSpecificationUtils;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
 import org.apache.asterix.metadata.entities.Feed;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedTrackingManager.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedTrackingManager.java
similarity index 98%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedTrackingManager.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedTrackingManager.java
index a1c6fb9c..29230c1 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedTrackingManager.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedTrackingManager.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
 
 import java.util.Arrays;
 import java.util.BitSet;
@@ -32,7 +32,6 @@
 import org.apache.asterix.external.feed.management.FeedConnectionId;
 import org.apache.asterix.external.feed.message.FeedTupleCommitAckMessage;
 import org.apache.asterix.external.feed.message.FeedTupleCommitResponseMessage;
-import org.apache.asterix.file.FeedOperations;
 import org.apache.hyracks.api.job.JobSpecification;
 
 public class FeedTrackingManager implements IFeedTrackingManager {
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkCollection.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java
similarity index 99%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkCollection.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java
index 9d746c8..53b9792 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkCollection.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
 
 import java.io.PrintWriter;
 import java.util.ArrayList;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkRequestResponseHandler.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkRequestResponseHandler.java
similarity index 99%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkRequestResponseHandler.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkRequestResponseHandler.java
index b30d8a7..2dc1162 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedWorkRequestResponseHandler.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkRequestResponseHandler.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
 
 import java.util.ArrayList;
 import java.util.HashMap;
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/FeedsActivator.java b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedsActivator.java
similarity index 98%
rename from asterix-app/src/main/java/org/apache/asterix/feed/FeedsActivator.java
rename to asterix-app/src/main/java/org/apache/asterix/app/external/FeedsActivator.java
index dc02a53..5a6d28e 100644
--- a/asterix-app/src/main/java/org/apache/asterix/feed/FeedsActivator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/app/external/FeedsActivator.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.feed;
+package org.apache.asterix.app.external;
 
 import java.io.PrintWriter;
 import java.util.ArrayList;
diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 46ea72b..9f024e9 100644
--- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -42,6 +42,11 @@
 import org.apache.asterix.api.common.APIFramework;
 import org.apache.asterix.api.common.SessionConfig;
 import org.apache.asterix.api.common.SessionConfig.OutputFormat;
+import org.apache.asterix.app.external.CentralFeedManager;
+import org.apache.asterix.app.external.ExternalIndexingOperations;
+import org.apache.asterix.app.external.FeedJoint;
+import org.apache.asterix.app.external.FeedLifecycleListener;
+import org.apache.asterix.app.external.FeedOperations;
 import org.apache.asterix.common.config.AsterixExternalProperties;
 import org.apache.asterix.common.config.DatasetConfig.DatasetType;
 import org.apache.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
@@ -69,13 +74,8 @@
 import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
 import org.apache.asterix.external.feed.watch.FeedActivity.FeedActivityDetails;
 import org.apache.asterix.external.indexing.ExternalFile;
-import org.apache.asterix.feed.CentralFeedManager;
-import org.apache.asterix.feed.FeedJoint;
-import org.apache.asterix.feed.FeedLifecycleListener;
 import org.apache.asterix.file.DatasetOperations;
 import org.apache.asterix.file.DataverseOperations;
-import org.apache.asterix.file.ExternalIndexingOperations;
-import org.apache.asterix.file.FeedOperations;
 import org.apache.asterix.file.IndexOperations;
 import org.apache.asterix.formats.nontagged.AqlTypeTraitProvider;
 import org.apache.asterix.lang.aql.statement.SubscribeFeedStatement;
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
index 54cac09..24df771 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
@@ -24,6 +24,7 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.asterix.app.external.ExternalIndexingOperations;
 import org.apache.asterix.common.config.AsterixStorageProperties;
 import org.apache.asterix.common.config.DatasetConfig.DatasetType;
 import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
index 83f91a0..be1c356 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
@@ -20,6 +20,7 @@
 
 import java.util.List;
 
+import org.apache.asterix.app.external.ExternalIndexingOperations;
 import org.apache.asterix.common.api.ILocalResourceMetadata;
 import org.apache.asterix.common.config.AsterixStorageProperties;
 import org.apache.asterix.common.config.DatasetConfig.DatasetType;
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
index adf0a4d..e683ef4 100644
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
@@ -34,6 +34,9 @@
 import org.apache.asterix.api.http.servlet.ShutdownAPIServlet;
 import org.apache.asterix.api.http.servlet.UpdateAPIServlet;
 import org.apache.asterix.api.http.servlet.VersionAPIServlet;
+import org.apache.asterix.app.external.CentralFeedManager;
+import org.apache.asterix.app.external.ExternalLibraryUtils;
+import org.apache.asterix.app.external.FeedLifecycleListener;
 import org.apache.asterix.common.api.AsterixThreadFactory;
 import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
 import org.apache.asterix.common.config.AsterixExternalProperties;
@@ -43,8 +46,6 @@
 import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
 import org.apache.asterix.event.service.ILookupService;
 import org.apache.asterix.external.feed.api.ICentralFeedManager;
-import org.apache.asterix.feed.CentralFeedManager;
-import org.apache.asterix.feed.FeedLifecycleListener;
 import org.apache.asterix.messaging.CCMessageBroker;
 import org.apache.asterix.metadata.MetadataManager;
 import org.apache.asterix.metadata.api.IAsterixStateProxy;
@@ -110,7 +111,7 @@
         setupFeedServer(externalProperties);
         feedServer.start();
 
-        ExternalLibraryBootstrap.setUpExternaLibraries(false);
+        ExternalLibraryUtils.setUpExternaLibraries(false);
         centralFeedManager = CentralFeedManager.getInstance();
         centralFeedManager.start();
 
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/FeedBootstrap.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/FeedBootstrap.java
index d5f1a51..a6be075 100644
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/FeedBootstrap.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/FeedBootstrap.java
@@ -18,9 +18,9 @@
  */
 package org.apache.asterix.hyracks.bootstrap;
 
+import org.apache.asterix.app.external.CentralFeedManager;
 import org.apache.asterix.common.config.MetadataConstants;
 import org.apache.asterix.external.util.FeedConstants;
-import org.apache.asterix.feed.CentralFeedManager;
 import org.apache.asterix.om.types.BuiltinType;
 import org.apache.asterix.om.types.IAType;
 
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
index 2bac1cf..8132d4b 100644
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
@@ -23,6 +23,8 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import org.apache.asterix.app.external.CentralFeedManager;
+import org.apache.asterix.app.external.ExternalIndexingOperations;
 import org.apache.asterix.common.api.IClusterManagementWork;
 import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
 import org.apache.asterix.common.api.IClusterManagementWorkResponse;
@@ -32,8 +34,6 @@
 import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
 import org.apache.asterix.common.config.MetadataConstants;
 import org.apache.asterix.external.indexing.ExternalFile;
-import org.apache.asterix.feed.CentralFeedManager;
-import org.apache.asterix.file.ExternalIndexingOperations;
 import org.apache.asterix.metadata.MetadataManager;
 import org.apache.asterix.metadata.MetadataTransactionContext;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index fcb196d..4922ae6 100644
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -26,6 +26,7 @@
 import java.util.logging.Logger;
 
 import org.apache.asterix.api.common.AsterixAppRuntimeContext;
+import org.apache.asterix.app.external.ExternalLibraryUtils;
 import org.apache.asterix.common.api.AsterixThreadFactory;
 import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
 import org.apache.asterix.common.config.AsterixMetadataProperties;
@@ -212,7 +213,7 @@
         if (isMetadataNode && !pendingFailbackCompletion) {
             runtimeContext.initializeMetadata(systemState == SystemState.NEW_UNIVERSE);
         }
-        ExternalLibraryBootstrap.setUpExternaLibraries(isMetadataNode && !pendingFailbackCompletion);
+        ExternalLibraryUtils.setUpExternaLibraries(isMetadataNode && !pendingFailbackCompletion);
 
         if (LOGGER.isLoggable(Level.INFO)) {
             LOGGER.info("Starting lifecycle components");