[NO ISSUE] Fix UDF Metadata Transactions

- Move UDF Metadata Transactions to be initiated from the CC
directly
- Remove old UDF initialization code
- Make DROP DATAVERSE remove libraries
- Make NCs properly read libraries on init with multipart dataverse
names

Change-Id: Ibda23a2e8308937f343d80eff04ede9a235da0d2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/4923
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/UdfApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/UdfApiServlet.java
index 54a62df..0f5d542 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/UdfApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/UdfApiServlet.java
@@ -19,21 +19,34 @@
 package org.apache.asterix.api.http.server;
 
 import java.io.File;
+import java.io.PrintWriter;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
+import java.rmi.RemoteException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.ConcurrentMap;
 
 import org.apache.asterix.app.external.ExternalLibraryUtils;
-import org.apache.asterix.app.message.DeleteUdfMessage;
 import org.apache.asterix.app.message.LoadUdfMessage;
 import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.exceptions.ErrorCode;
 import org.apache.asterix.common.messaging.api.ICCMessageBroker;
 import org.apache.asterix.common.messaging.api.INcAddressedMessage;
 import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.IMetadataLockUtil;
+import org.apache.asterix.common.metadata.LockList;
+import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.MetadataTransactionContext;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.DatasourceAdapter;
+import org.apache.asterix.metadata.entities.Dataverse;
+import org.apache.asterix.metadata.entities.Function;
+import org.apache.asterix.metadata.entities.Library;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.common.utils.Pair;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.deployment.DeploymentId;
@@ -75,6 +88,8 @@
 
     @Override
     protected void post(IServletRequest request, IServletResponse response) {
+
+        PrintWriter responseWriter = response.writer();
         FullHttpRequest req = request.getHttpRequest();
         Pair<String, DataverseName> resourceNames;
         try {
@@ -85,8 +100,17 @@
         }
         String resourceName = resourceNames.first;
         DataverseName dataverse = resourceNames.second;
+        IMetadataLockUtil mdLockUtil = appCtx.getMetadataLockUtil();
+        MetadataTransactionContext mdTxnCtx = null;
+        LockList mdLockList = null;
         File udf = null;
         try {
+            MetadataManager.INSTANCE.init();
+            mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+            MetadataProvider metadataProvider = MetadataProvider.create(appCtx, null);
+            mdLockList = metadataProvider.getLocks();
+            mdLockUtil.createLibraryBegin(appCtx.getMetadataLockManager(), metadataProvider.getLocks(), dataverse,
+                    resourceName);
             File workingDir = new File(appCtx.getServiceContext().getServerCtx().getBaseDir().getAbsolutePath(),
                     UDF_TMP_DIR_PREFIX);
             if (!workingDir.exists()) {
@@ -102,35 +126,104 @@
                     fc.write(content);
                 }
             }
-            IHyracksClientConnection hcc = appCtx.getHcc();
-            DeploymentId udfName = new DeploymentId(makeDeploymentId(dataverse, resourceName));
-            ClassLoader cl = appCtx.getLibraryManager().getLibraryClassLoader(dataverse, resourceName);
-            if (cl != null) {
-                deleteUdf(dataverse, resourceName);
-            }
-            hcc.deployBinary(udfName, Arrays.asList(udf.toString()), true);
-            ExternalLibraryUtils.setUpExternaLibrary(appCtx.getLibraryManager(), false,
-                    FileUtil.joinPath(appCtx.getServiceContext().getServerCtx().getBaseDir().getAbsolutePath(),
-                            "applications", udfName.toString()));
-
-            long reqId = broker.newRequestId();
-            List<INcAddressedMessage> requests = new ArrayList<>();
-            List<String> ncs = new ArrayList<>(appCtx.getClusterStateManager().getParticipantNodes());
-            ncs.forEach(s -> requests.add(new LoadUdfMessage(dataverse, resourceName, reqId)));
-            broker.sendSyncRequestToNCs(reqId, ncs, requests, UDF_RESPONSE_TIMEOUT);
+            setupBinariesAndClassloaders(dataverse, resourceName, udf);
+            installLibrary(mdTxnCtx, dataverse, resourceName);
+            MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
         } catch (Exception e) {
+            try {
+                ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverse, resourceName);
+            } catch (Exception e2) {
+                e.addSuppressed(e2);
+            }
             response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
+            responseWriter.write(e.getMessage());
+            responseWriter.flush();
             LOGGER.error(e);
-            return;
-        } finally {
+            if (mdTxnCtx != null) {
+                try {
+                    MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
+                } catch (RemoteException r) {
+                    LOGGER.error("Unable to abort metadata transaction", r);
+                }
+            }
             if (udf != null) {
                 udf.delete();
             }
+            return;
+        } finally {
+            if (mdLockList != null) {
+                mdLockList.unlock();
+            }
         }
         response.setStatus(HttpResponseStatus.OK);
 
     }
 
+    private void setupBinariesAndClassloaders(DataverseName dataverse, String resourceName, File udf) throws Exception {
+        IHyracksClientConnection hcc = appCtx.getHcc();
+        DeploymentId udfName = new DeploymentId(makeDeploymentId(dataverse, resourceName));
+        ClassLoader cl = appCtx.getLibraryManager().getLibraryClassLoader(dataverse, resourceName);
+        if (cl != null) {
+            //prepare to replace the binary
+            ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverse, resourceName);
+        }
+        hcc.deployBinary(udfName, Arrays.asList(udf.toString()), true);
+        //setup for CC
+        ExternalLibraryUtils.setUpExternaLibrary(appCtx.getLibraryManager(),
+                FileUtil.joinPath(appCtx.getServiceContext().getServerCtx().getBaseDir().getAbsolutePath(),
+                        "applications", udfName.toString()));
+        //setup NCs
+        long reqId = broker.newRequestId();
+        List<INcAddressedMessage> requests = new ArrayList<>();
+        List<String> ncs = new ArrayList<>(appCtx.getClusterStateManager().getParticipantNodes());
+        ncs.forEach(s -> requests.add(new LoadUdfMessage(dataverse, resourceName, reqId)));
+        broker.sendSyncRequestToNCs(reqId, ncs, requests, UDF_RESPONSE_TIMEOUT);
+    }
+
+    private static void installLibrary(MetadataTransactionContext mdTxnCtx, DataverseName dataverse, String libraryName)
+            throws RemoteException, AlgebricksException {
+        Library libraryInMetadata = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName);
+        // Get the dataverse
+        Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse);
+        if (dv == null) {
+            throw new AsterixException(ErrorCode.UNKNOWN_DATAVERSE);
+        }
+        if (libraryInMetadata != null) {
+            //replacing binary, library already exists
+            return;
+        }
+        // Add library
+        MetadataManager.INSTANCE.addLibrary(mdTxnCtx, new Library(dataverse, libraryName));
+        if (LOGGER.isInfoEnabled()) {
+            LOGGER.info("Added library " + libraryName + " to Metadata");
+        }
+    }
+
+    private static void deleteLibrary(MetadataTransactionContext mdTxnCtx, DataverseName dataverse, String libraryName)
+            throws RemoteException, AlgebricksException {
+        Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse);
+        if (dv == null) {
+            throw new AsterixException(ErrorCode.UNKNOWN_DATAVERSE);
+        }
+        Library library = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName);
+        if (library == null) {
+            throw new AsterixException(ErrorCode.UNKNOWN_LIBRARY);
+        }
+        List<Function> functions = MetadataManager.INSTANCE.getDataverseFunctions(mdTxnCtx, dataverse);
+        for (Function function : functions) {
+            if (libraryName.equals(function.getLibrary())) {
+                throw new AsterixException(ErrorCode.METADATA_DROP_LIBRARY_IN_USE, libraryName);
+            }
+        }
+        List<DatasourceAdapter> adapters = MetadataManager.INSTANCE.getDataverseAdapters(mdTxnCtx, dataverse);
+        for (DatasourceAdapter adapter : adapters) {
+            if (libraryName.equals(adapter.getLibrary())) {
+                throw new AsterixException(ErrorCode.METADATA_DROP_LIBRARY_IN_USE, libraryName);
+            }
+        }
+        MetadataManager.INSTANCE.dropLibrary(mdTxnCtx, dataverse, libraryName);
+    }
+
     public static String makeDeploymentId(DataverseName dv, String resourceName) {
         List<String> dvParts = dv.getParts();
         dvParts.add(resourceName);
@@ -138,16 +231,6 @@
         return dvWithLibrarySuffix.getCanonicalForm();
     }
 
-    private void deleteUdf(DataverseName dataverse, String resourceName) throws Exception {
-        long reqId = broker.newRequestId();
-        List<INcAddressedMessage> requests = new ArrayList<>();
-        List<String> ncs = new ArrayList<>(appCtx.getClusterStateManager().getParticipantNodes());
-        ncs.forEach(s -> requests.add(new DeleteUdfMessage(dataverse, resourceName, reqId)));
-        broker.sendSyncRequestToNCs(reqId, ncs, requests, UDF_RESPONSE_TIMEOUT);
-        appCtx.getLibraryManager().deregisterLibraryClassLoader(dataverse, resourceName);
-        appCtx.getHcc().unDeployBinary(new DeploymentId(makeDeploymentId(dataverse, resourceName)));
-    }
-
     @Override
     protected void delete(IServletRequest request, IServletResponse response) {
         Pair<String, DataverseName> resourceNames;
@@ -157,14 +240,37 @@
             response.setStatus(HttpResponseStatus.BAD_REQUEST);
             return;
         }
+        PrintWriter responseWriter = response.writer();
         String resourceName = resourceNames.first;
         DataverseName dataverse = resourceNames.second;
+        IMetadataLockUtil mdLockUtil = appCtx.getMetadataLockUtil();
+        MetadataTransactionContext mdTxnCtx = null;
+        LockList mdLockList = null;
         try {
-            deleteUdf(dataverse, resourceName);
+            MetadataManager.INSTANCE.init();
+            mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+            MetadataProvider metadataProvider = MetadataProvider.create(appCtx, null);
+            mdLockList = metadataProvider.getLocks();
+            mdLockUtil.dropLibraryBegin(appCtx.getMetadataLockManager(), metadataProvider.getLocks(), dataverse,
+                    resourceName);
+            deleteLibrary(mdTxnCtx, dataverse, resourceName);
+            ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverse, resourceName);
+            MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
         } catch (Exception e) {
+            try {
+                MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
+            } catch (RemoteException r) {
+                LOGGER.error("Unable to abort metadata transaction", r);
+            }
             response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
+            responseWriter.write(e.getMessage());
+            responseWriter.flush();
             LOGGER.error(e);
             return;
+        } finally {
+            if (mdLockList != null) {
+                mdLockList.unlock();
+            }
         }
         response.setStatus(HttpResponseStatus.OK);
     }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
index 758bd45..47a460e 100755
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
@@ -18,31 +18,23 @@
  */
 package org.apache.asterix.app.external;
 
+import static org.apache.asterix.api.http.server.UdfApiServlet.UDF_RESPONSE_TIMEOUT;
+import static org.apache.asterix.api.http.server.UdfApiServlet.makeDeploymentId;
+
 import java.io.File;
 import java.io.FilenameFilter;
-import java.io.IOException;
 import java.net.URL;
 import java.net.URLClassLoader;
-import java.rmi.RemoteException;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
-import org.apache.asterix.common.exceptions.ACIDException;
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.app.message.DeleteUdfMessage;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
 import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.common.messaging.api.ICCMessageBroker;
+import org.apache.asterix.common.messaging.api.INcAddressedMessage;
 import org.apache.asterix.common.metadata.DataverseName;
-import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.metadata.MetadataTransactionContext;
-import org.apache.asterix.metadata.entities.DatasourceAdapter;
-import org.apache.asterix.metadata.entities.Dataverse;
-import org.apache.asterix.metadata.entities.Function;
-import org.apache.asterix.metadata.entities.Library;
-import org.apache.asterix.metadata.utils.MetadataUtil;
-import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
-import org.apache.logging.log4j.Level;
+import org.apache.hyracks.api.deployment.DeploymentId;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -54,30 +46,17 @@
     private ExternalLibraryUtils() {
     }
 
-    public static void setUpExternaLibrary(ILibraryManager externalLibraryManager, boolean isMetadataNode,
-            String libraryPath) throws Exception {
-        // start by un-installing removed libraries (Metadata Node only)
-        Map<DataverseName, List<String>> uninstalledLibs = null;
-        if (isMetadataNode) {
-            uninstalledLibs = uninstallLibraries();
-        }
-
-        // get the directory of the to be installed libraries
-        String[] pathSplit = libraryPath.split("\\.");
-        String[] dvSplit = pathSplit[pathSplit.length - 2].split("/");
-        DataverseName dataverse = DataverseName.createSinglePartName(dvSplit[dvSplit.length - 1]); //TODO(MULTI_PART_DATAVERSE_NAME):REVISIT
-        String name = pathSplit[pathSplit.length - 1].trim();
-        File installLibDir = new File(libraryPath);
-
-        // directory exists?
-        if (installLibDir.exists()) {
-            registerClassLoader(externalLibraryManager, dataverse, name, libraryPath);
-            configureLibrary(externalLibraryManager, dataverse, name, installLibDir, uninstalledLibs, isMetadataNode);
-        }
+    public static void setUpExternaLibrary(ILibraryManager externalLibraryManager, String libraryPath)
+            throws Exception {
+        // get the installed library dirs
+        String[] parts = libraryPath.split(File.separator);
+        DataverseName catenatedDv = DataverseName.createFromCanonicalForm(parts[parts.length - 1]);
+        String libraryName = catenatedDv.getParts().get(catenatedDv.getParts().size() - 1);
+        DataverseName dvName = DataverseName.create(catenatedDv.getParts(), 0, catenatedDv.getParts().size() - 1);
+        registerClassLoader(externalLibraryManager, dvName, libraryName, libraryPath);
     }
 
-    public static void setUpInstalledLibraries(ILibraryManager externalLibraryManager, boolean isMetadataNode,
-            File appDir) throws Exception {
+    public static void setUpInstalledLibraries(ILibraryManager externalLibraryManager, File appDir) throws Exception {
         File[] libs = appDir.listFiles(new FilenameFilter() {
             @Override
             public boolean accept(File dir, String name) {
@@ -86,170 +65,20 @@
         });
         if (libs != null) {
             for (File lib : libs) {
-                setUpExternaLibrary(externalLibraryManager, isMetadataNode, lib.getAbsolutePath());
+                setUpExternaLibrary(externalLibraryManager, lib.getAbsolutePath());
             }
         }
     }
 
-    /**
-     * un-install libraries.
-     *
-     * @return a map from dataverse -> list of uninstalled libraries.
-     * @throws Exception
-     */
-    private static Map<DataverseName, List<String>> uninstallLibraries() throws Exception {
-        Map<DataverseName, List<String>> uninstalledLibs = new HashMap<>();
-        // get the directory of the un-install libraries
-        File uninstallLibDir = getLibraryUninstallDir();
-        String[] uninstallLibNames;
-        // directory exists?
-        if (uninstallLibDir.exists()) {
-            // list files
-            uninstallLibNames = uninstallLibDir.list(nonHiddenFileNameFilter);
-            for (String uninstallLibName : uninstallLibNames) {
-                // Get the <dataverse name - library name> pair
-                String[] components = uninstallLibName.split("\\.");
-                DataverseName dataverse = DataverseName.createSinglePartName(components[0]); //TODO(MULTI_PART_DATAVERSE_NAME):REVISIT
-                String libName = components[1];
-                // un-install
-                uninstallLibrary(dataverse, libName);
-                // delete the library file
-                new File(uninstallLibDir, uninstallLibName).delete();
-                // add the library to the list of uninstalled libraries
-                List<String> uinstalledLibsInDv = uninstalledLibs.get(dataverse);
-                if (uinstalledLibsInDv == null) {
-                    uinstalledLibsInDv = new ArrayList<>();
-                    uninstalledLibs.put(dataverse, uinstalledLibsInDv);
-                }
-                uinstalledLibsInDv.add(libName);
-            }
-        }
-        return uninstalledLibs;
-    }
-
-    /**
-     * Remove the library from metadata completely.
-     * TODO Currently, external libraries only include functions and adapters. we need to extend this to include:
-     * 1. external data source
-     * 2. data parser
-     *
-     * @param dataverse
-     * @param libraryName
-     * @return true if the library was found and removed, false otherwise
-     * @throws AsterixException
-     * @throws RemoteException
-     * @throws ACIDException
-     */
-    public static boolean uninstallLibrary(DataverseName dataverse, String libraryName)
-            throws AsterixException, RemoteException, ACIDException {
-        MetadataTransactionContext mdTxnCtx = null;
-        try {
-            // begin transaction
-            mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
-            // make sure dataverse exists
-            Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse);
-            if (dv == null) {
-                return false;
-            }
-            // make sure library exists
-            Library library = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName);
-            if (library == null) {
-                return false;
-            }
-
-            // get dataverse functions
-            List<Function> functions = MetadataManager.INSTANCE.getDataverseFunctions(mdTxnCtx, dataverse);
-            for (Function function : functions) {
-                // does function belong to library?
-                if (function.getName().startsWith(libraryName + "#")) {
-                    // drop the function
-                    MetadataManager.INSTANCE.dropFunction(mdTxnCtx,
-                            new FunctionSignature(dataverse, function.getName(), function.getArity()));
-                }
-            }
-
-            // get the dataverse adapters
-            List<DatasourceAdapter> adapters = MetadataManager.INSTANCE.getDataverseAdapters(mdTxnCtx, dataverse);
-            for (DatasourceAdapter adapter : adapters) {
-                // belong to the library?
-                if (adapter.getAdapterIdentifier().getName().startsWith(libraryName + "#")) {
-                    // remove adapter <! we didn't check if there are feeds which use this adapter>
-                    MetadataManager.INSTANCE.dropAdapter(mdTxnCtx, dataverse, adapter.getAdapterIdentifier().getName());
-                }
-            }
-            // drop the library itself
-            MetadataManager.INSTANCE.dropLibrary(mdTxnCtx, dataverse, libraryName);
-            MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
-        } catch (Exception e) {
-            MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
-            throw new AsterixException(e);
-        }
-        return true;
-    }
-
-    private static void addLibraryToMetadata(Map<DataverseName, List<String>> uninstalledLibs, DataverseName dataverse,
-            String libraryName) throws ACIDException, RemoteException {
-        // Modify metadata accordingly
-        List<String> uninstalledLibsInDv = uninstalledLibs.get(dataverse);
-        // was this library just un-installed?
-        boolean wasUninstalled = uninstalledLibsInDv != null && uninstalledLibsInDv.contains(libraryName);
-        MetadataTransactionContext mdTxnCtx = null;
-        try {
-            mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
-            Library libraryInMetadata = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName);
-            if (libraryInMetadata != null && !wasUninstalled) {
-                // exists in metadata and was not un-installed, we return.
-                // Another place which shows that our metadata transactions are broken
-                // (we didn't call commit before!!!)
-                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
-                return;
-            }
-
-            // Add library
-            MetadataManager.INSTANCE.addLibrary(mdTxnCtx, new Library(dataverse, libraryName));
-            if (LOGGER.isInfoEnabled()) {
-                LOGGER.info("Added library " + libraryName + " to Metadata");
-            }
-
-            // Get the dataverse
-            Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse);
-            if (dv == null) {
-                MetadataManager.INSTANCE.addDataverse(mdTxnCtx, new Dataverse(dataverse,
-                        NonTaggedDataFormat.NON_TAGGED_DATA_FORMAT, MetadataUtil.PENDING_NO_OP));
-            }
-
-            MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
-        } catch (Exception e) {
-            if (LOGGER.isErrorEnabled()) {
-                LOGGER.log(Level.ERROR, "Exception in installing library " + libraryName, e);
-            }
-            MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
-        }
-    }
-
-    /**
-     * Each element of a library is installed as part of a transaction. Any
-     * failure in installing an element does not effect installation of other
-     * libraries.
-     */
-    protected static void configureLibrary(ILibraryManager libraryManager, DataverseName dataverse, String libraryName,
-            final File libraryDir, Map<DataverseName, List<String>> uninstalledLibs, boolean isMetadataNode)
-            throws Exception {
-
-        String[] libraryDescriptors = libraryDir.list((dir, name) -> name.endsWith(".xml"));
-
-        if (libraryDescriptors == null) {
-            throw new IOException("Unable to list files in directory " + libraryDir);
-        }
-
-        if (libraryDescriptors.length > 1) {
-            throw new IllegalStateException("More than 1 library descriptors defined");
-        }
-
-        // Prepare possible parameters
-        if (isMetadataNode) {
-            addLibraryToMetadata(uninstalledLibs, dataverse, libraryName);
-        }
+    public static void deleteDeployedUdf(ICCMessageBroker broker, ICcApplicationContext appCtx,
+            DataverseName dataverseName, String lib) throws Exception {
+        long reqId = broker.newRequestId();
+        List<INcAddressedMessage> requests = new ArrayList<>();
+        List<String> ncs = new ArrayList<>(appCtx.getClusterStateManager().getParticipantNodes());
+        ncs.forEach(s -> requests.add(new DeleteUdfMessage(dataverseName, lib, reqId)));
+        broker.sendSyncRequestToNCs(reqId, ncs, requests, UDF_RESPONSE_TIMEOUT);
+        appCtx.getLibraryManager().deregisterLibraryClassLoader(dataverseName, lib);
+        appCtx.getHcc().unDeployBinary(new DeploymentId(makeDeploymentId(dataverseName, lib)));
     }
 
     /**
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AbstractUdfMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AbstractUdfMessage.java
index 1c212fe..b8c77e7 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AbstractUdfMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AbstractUdfMessage.java
@@ -46,12 +46,9 @@
     @Override
     public void handle(INcApplicationContext appCtx) {
         ILibraryManager mgr = appCtx.getLibraryManager();
-        String mdNodeName = appCtx.getMetadataProperties().getMetadataNodeName();
-        String nodeName = appCtx.getServiceContext().getNodeId();
         INCMessageBroker broker = (INCMessageBroker) appCtx.getServiceContext().getMessageBroker();
-        boolean isMdNode = mdNodeName.equals(nodeName);
         try {
-            handleAction(mgr, isMdNode, appCtx);
+            handleAction(mgr, appCtx);
             broker.sendMessageToCC(getCcId(), new UdfResponseMessage(reqId, null));
         } catch (Exception e) {
             try {
@@ -64,7 +61,6 @@
 
     }
 
-    protected abstract void handleAction(ILibraryManager mgr, boolean isMdNode, INcApplicationContext appCtx)
-            throws Exception;
+    protected abstract void handleAction(ILibraryManager mgr, INcApplicationContext appCtx) throws Exception;
 
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DeleteUdfMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DeleteUdfMessage.java
index 3fb58fa..7bcfa9b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DeleteUdfMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DeleteUdfMessage.java
@@ -18,7 +18,6 @@
  */
 package org.apache.asterix.app.message;
 
-import org.apache.asterix.app.external.ExternalLibraryUtils;
 import org.apache.asterix.common.api.INcApplicationContext;
 import org.apache.asterix.common.library.ILibraryManager;
 import org.apache.asterix.common.metadata.DataverseName;
@@ -32,10 +31,7 @@
     }
 
     @Override
-    protected void handleAction(ILibraryManager mgr, boolean isMdNode, INcApplicationContext appCtx) throws Exception {
-        if (isMdNode) {
-            ExternalLibraryUtils.uninstallLibrary(dataverseName, libraryName);
-        }
+    protected void handleAction(ILibraryManager mgr, INcApplicationContext appCtx) {
         mgr.deregisterLibraryClassLoader(dataverseName, libraryName);
     }
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/LoadUdfMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/LoadUdfMessage.java
index 6dffb28..7ccdfd4 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/LoadUdfMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/LoadUdfMessage.java
@@ -33,8 +33,8 @@
     }
 
     @Override
-    protected void handleAction(ILibraryManager mgr, boolean isMdNode, INcApplicationContext appCtx) throws Exception {
-        ExternalLibraryUtils.setUpExternaLibrary(mgr, isMdNode,
+    protected void handleAction(ILibraryManager mgr, INcApplicationContext appCtx) throws Exception {
+        ExternalLibraryUtils.setUpExternaLibrary(mgr,
                 FileUtil.joinPath(appCtx.getServiceContext().getServerCtx().getBaseDir().getAbsolutePath(),
                         "applications", dataverseName.getCanonicalForm() + "." + libraryName)); //TODO(MULTI_PART_DATAVERSE_NAME):REVISIT
     }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/ExternalLibrarySetupTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/ExternalLibrarySetupTask.java
index 1ca2b78..40262ac 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/ExternalLibrarySetupTask.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/ExternalLibrarySetupTask.java
@@ -38,7 +38,7 @@
     public void perform(CcId ccId, IControllerService cs) throws HyracksDataException {
         INcApplicationContext appContext = (INcApplicationContext) cs.getApplicationContext();
         try {
-            ExternalLibraryUtils.setUpInstalledLibraries(appContext.getLibraryManager(), metadataNode,
+            ExternalLibraryUtils.setUpInstalledLibraries(appContext.getLibraryManager(),
                     cs.getContext().getServerCtx().getAppDir());
         } catch (Exception e) {
             throw HyracksDataException.create(e);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 16ca37f..814c96e 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -50,6 +50,7 @@
 import org.apache.asterix.app.active.ActiveEntityEventsListener;
 import org.apache.asterix.app.active.ActiveNotificationHandler;
 import org.apache.asterix.app.active.FeedEventsListener;
+import org.apache.asterix.app.external.ExternalLibraryUtils;
 import org.apache.asterix.app.result.ExecutionError;
 import org.apache.asterix.app.result.ResultHandle;
 import org.apache.asterix.app.result.ResultReader;
@@ -78,6 +79,7 @@
 import org.apache.asterix.common.exceptions.WarningCollector;
 import org.apache.asterix.common.exceptions.WarningUtil;
 import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.common.messaging.api.ICCMessageBroker;
 import org.apache.asterix.common.metadata.DataverseName;
 import org.apache.asterix.common.metadata.IMetadataLockUtil;
 import org.apache.asterix.common.utils.JobUtils;
@@ -1307,6 +1309,8 @@
         boolean bActiveTxn = true;
         metadataProvider.setMetadataTxnContext(mdTxnCtx);
         List<JobSpecification> jobsToExecute = new ArrayList<>();
+        List<Library> librariesToDelete = new ArrayList<>();
+        ICCMessageBroker broker = (ICCMessageBroker) appCtx.getServiceContext().getMessageBroker();
         try {
             Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName);
             if (dv == null) {
@@ -1326,6 +1330,7 @@
                             function.getDataverseName() + "." + function.getName() + "@" + function.getArity());
                 }
             }
+
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
             bActiveTxn = false;
             // # disconnect all feeds from any datasets in the dataverse.
@@ -1381,6 +1386,7 @@
             }
             jobsToExecute.add(DataverseUtil.dropDataverseJobSpec(dv, metadataProvider));
 
+            librariesToDelete = MetadataManager.INSTANCE.getDataverseLibraries(mdTxnCtx, dataverseName);
             // #. mark PendingDropOp on the dataverse record by
             // first, deleting the dataverse record from the DATAVERSE_DATASET
             // second, inserting the dataverse record with the PendingDropOp value into the
@@ -1397,6 +1403,10 @@
                 runJob(hcc, jobSpec);
             }
 
+            for (Library lib : librariesToDelete) {
+                ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverseName, lib.getName());
+            }
+
             mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
             bActiveTxn = true;
             metadataProvider.setMetadataTxnContext(mdTxnCtx);
@@ -1434,6 +1444,10 @@
                     for (JobSpecification jobSpec : jobsToExecute) {
                         runJob(hcc, jobSpec);
                     }
+
+                    for (Library lib : librariesToDelete) {
+                        ExternalLibraryUtils.deleteDeployedUdf(broker, appCtx, dataverseName, lib.getName());
+                    }
                 } catch (Exception e2) {
                     // do no throw exception since still the metadata needs to be compensated.
                     e.addSuppressed(e2);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
index bc10d4f..ae85381 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
@@ -157,7 +157,7 @@
         ReplicationProperties repProp =
                 new ReplicationProperties(PropertiesAccessor.getInstance(ccServiceCtx.getAppConfig()));
         INcLifecycleCoordinator lifecycleCoordinator = createNcLifeCycleCoordinator(repProp.isReplicationEnabled());
-        ExternalLibraryUtils.setUpInstalledLibraries(libraryManager, false, ccServiceCtx.getServerCtx().getAppDir());
+        ExternalLibraryUtils.setUpInstalledLibraries(libraryManager, ccServiceCtx.getServerCtx().getAppDir());
         componentProvider = new StorageComponentProvider();
 
         ccExtensionManager = new CCExtensionManager(new ArrayList<>(getExtensions()));
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
index fd40a01..0c3ac81 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
@@ -23,6 +23,7 @@
 import java.net.URL;
 
 import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.commons.io.IOUtils;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
 import org.apache.http.auth.AuthScope;
@@ -39,7 +40,6 @@
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.hyracks.algebricks.common.utils.Pair;
-import org.apache.hyracks.api.exceptions.HyracksException;
 
 @SuppressWarnings("squid:S134")
 public class ExternalUDFLibrarian implements IExternalUDFLibrarian {
@@ -75,7 +75,7 @@
         HttpResponse response = hc.execute(post, hcCtx);
         response.getEntity().consumeContent();
         if (response.getStatusLine().getStatusCode() != 200) {
-            throw new HyracksException(response.getStatusLine().toString());
+            throw new AsterixException(response.getStatusLine().toString());
         }
     }
 
@@ -93,9 +93,17 @@
         hcCtx.setAuthCache(ac);
         HttpDelete del = new HttpDelete(url.toString());
         HttpResponse response = hc.execute(del, hcCtx);
+        String resp = null;
+        int respCode = response.getStatusLine().getStatusCode();
+        if (respCode == 500) {
+            resp = IOUtils.toString(response.getEntity().getContent());
+        }
         response.getEntity().consumeContent();
-        if (response.getStatusLine().getStatusCode() != 200) {
-            throw new AsterixException(response.getStatusLine().toString());
+        if (resp == null && respCode != 200) {
+            resp = response.getStatusLine().toString();
+        }
+        if (resp != null) {
+            throw new AsterixException(resp);
         }
     }
 
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.6.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.6.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.lib.sqlpp
deleted file mode 100644
index 172bed4..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/deterministic/deterministic.5.lib.sqlpp
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-uninstall externallibtest testlib admin admin
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.5.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.lib.sqlpp
deleted file mode 100644
index 41880cb..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital/getCapital.4.lib.sqlpp
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-uninstall externallibtest testlib admin admin
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.4.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.5.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.4.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.4.lib.sqlpp
deleted file mode 100644
index 41880cb..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/getCapital_open/getCapital_open.4.lib.sqlpp
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-uninstall externallibtest testlib admin admin
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.4.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.4.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.7.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.7.lib.sqlpp
deleted file mode 100644
index 41880cb..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.7.lib.sqlpp
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-uninstall externallibtest testlib admin admin
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.0.ddl.sqlpp
similarity index 90%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.0.ddl.sqlpp
index 6b2fb1f..76cc70d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.0.ddl.sqlpp
@@ -16,4 +16,5 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-uninstall udfs testlib admin admin
+DROP DATAVERSE externallibtest if exists;
+CREATE DATAVERSE  externallibtest;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.1.lib.sqlpp
similarity index 88%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.1.lib.sqlpp
index 6b2fb1f..3dc6eb6 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.1.lib.sqlpp
@@ -16,4 +16,4 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-uninstall udfs testlib admin admin
+install externallibtest testlib admin admin target/data/externallib/asterix-external-data-testlib.zip
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.2.ddl.sqlpp
similarity index 83%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.2.ddl.sqlpp
index 6b2fb1f..fd815d1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.2.ddl.sqlpp
@@ -16,4 +16,6 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-uninstall udfs testlib admin admin
+ USE externallibtest;
+
+create function mysum(a: int32, b: int32) returns int32 language java as "testlib","org.apache.asterix.external.library.MySumFactory";
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.3.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.3.query.sqlpp
index 6b2fb1f..a23c197 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.3.query.sqlpp
@@ -16,4 +16,6 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-uninstall udfs testlib admin admin
+use externallibtest;
+
+mysum(9.7, 4);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.4.lib.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.4.lib.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.4.lib.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.5.ddl.sqlpp
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.5.ddl.sqlpp
index 6b2fb1f..f8e2ae7 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.5.ddl.sqlpp
@@ -16,4 +16,6 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-uninstall udfs testlib admin admin
+ USE externallibtest;
+
+drop function externallibtest.mysum@2;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.6.lib.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.4.lib.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.6.lib.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.7.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/my_array_sum/my_array_sum.5.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum_dropinuse/mysum_dropinuse.7.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.5.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.lib.sqlpp
deleted file mode 100644
index e44b339..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/type_validation/type_validation.4.lib.sqlpp
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-uninstall externallibtest testlib admin admin
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.4.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.5.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.4.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.4.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.4.lib.sqlpp
deleted file mode 100644
index 41880cb..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_metadata/udf_metadata.4.lib.sqlpp
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-uninstall externallibtest testlib admin admin
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.4.ddl.sqlpp
similarity index 95%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.4.ddl.sqlpp
index 6b2fb1f..1a46de9 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.lib.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/upperCase/upperCase.4.ddl.sqlpp
@@ -16,4 +16,4 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-uninstall udfs testlib admin admin
+ drop dataverse externallibtest;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.7.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.7.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feed-with-external-function/feed-with-external-function.6.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysum_dropinuse/mysum_dropinuse.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysum_dropinuse/mysum_dropinuse.1.adm
new file mode 100644
index 0000000..ca7bf83
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/mysum_dropinuse/mysum_dropinuse.1.adm
@@ -0,0 +1 @@
+13
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
index 91d4969..b6cf0cc 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
@@ -44,6 +44,12 @@
       </compilation-unit>
     </test-case>
     <test-case FilePath="external-library">
+      <compilation-unit name="mysum_dropinuse">
+        <output-dir compare="Text">mysum_dropinuse</output-dir>
+        <expected-error>Library testlib is being used. It cannot be dropped</expected-error>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="external-library">
       <compilation-unit name="my_array_sum">
         <output-dir compare="Text">my_array_sum</output-dir>
       </compilation-unit>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 3d30360..7fef69e 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -314,6 +314,7 @@
     public static final int FAILED_TO_PARSE_METADATA = 3115;
     public static final int INPUT_DECODE_FAILURE = 3116;
     public static final int FAILED_TO_PARSE_MALFORMED_LOG_RECORD = 3117;
+    public static final int METADATA_DROP_LIBRARY_IN_USE = 3118;
 
     // Lifecycle management errors
     public static final int DUPLICATE_PARTITION_ID = 4000;
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index fc356e9..4461ff1 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -312,6 +312,7 @@
 3115 = Failed to parse record metadata
 3116 = Failed to decode input
 3117 = Failed to parse record, malformed log record
+3118 = Library %1$s is being used. It cannot be dropped
 
 # Lifecycle management errors
 4000 = Partition id %1$s for node %2$s already in use by node %3$s
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index cf7562f..ca98328 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -534,6 +534,11 @@
                         true);
             }
 
+            //Drop libraries, similarly.
+            for (Library lib : getDataverseLibraries(txnId, dataverseName)) {
+                dropLibrary(txnId, lib.getDataverseName(), lib.getName());
+            }
+
             List<Dataset> dataverseDatasets;
             Dataset ds;
             dataverseDatasets = getDataverseDatasets(txnId, dataverseName);