Merge branch 'master' into westmann/scratch
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index 96d0617..3502abb 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -129,16 +129,26 @@
 
         isMetadataNode = nodeId.equals(metadataProperties.getMetadataNodeName());
         if (isMetadataNode) {
-            registerRemoteMetadataNode(proxy);
-
             if (LOGGER.isLoggable(Level.INFO)) {
                 LOGGER.info("Bootstrapping metadata");
             }
-            MetadataManager.INSTANCE = new MetadataManager(proxy, metadataProperties);
-            MetadataManager.INSTANCE.init();
+            MetadataNode.INSTANCE.initialize(runtimeContext);
+
+            // This is a special case, we just give the metadataNode directly.
+            // This way we can delay the registration of the metadataNode until
+            // it is completely initialized.
+            MetadataManager.INSTANCE = new MetadataManager(proxy, MetadataNode.INSTANCE);
             MetadataBootstrap.startUniverse(((IAsterixPropertiesProvider) runtimeContext), ncApplicationContext,
                     systemState == SystemState.NEW_UNIVERSE);
             MetadataBootstrap.startDDLRecovery();
+
+            IMetadataNode stub = null;
+            stub = (IMetadataNode) UnicastRemoteObject.exportObject(MetadataNode.INSTANCE, 0);
+            proxy.setMetadataNode(stub);
+            
+            if (LOGGER.isLoggable(Level.INFO)) {
+                LOGGER.info("Metadata node bound");
+            }
         }
 
         if (LOGGER.isLoggable(Level.INFO)) {
@@ -166,17 +176,6 @@
         // reclaim storage for orphaned index artifacts in NCs.
     }
 
-    public void registerRemoteMetadataNode(IAsterixStateProxy proxy) throws RemoteException {
-        IMetadataNode stub = null;
-        MetadataNode.INSTANCE.initialize(runtimeContext);
-        stub = (IMetadataNode) UnicastRemoteObject.exportObject(MetadataNode.INSTANCE, 0);
-        proxy.setMetadataNode(stub);
-
-        if (LOGGER.isLoggable(Level.INFO)) {
-            LOGGER.info("Metadata node bound");
-        }
-    }
-
     /**
      * Shutdown hook that invokes {@link NCApplicationEntryPoint#stop() stop} method.
      */
diff --git a/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java b/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java
index 3fec2c8..b121395 100644
--- a/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java
+++ b/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java
@@ -188,10 +188,16 @@
 
     private static String[] handleError(GetMethod method) throws Exception {
         String errorBody = method.getResponseBodyAsString();
-        JSONObject result = new JSONObject(errorBody);
-        String[] errors = { result.getJSONArray("error-code").getString(0), result.getString("summary"),
-                result.getString("stacktrace") };
-        return errors;
+        try {
+            JSONObject result = new JSONObject(errorBody);
+            String[] errors = { result.getJSONArray("error-code").getString(0), result.getString("summary"),
+                    result.getString("stacktrace") };
+            return errors;
+        } catch (JSONException je) {
+            GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, "Response body is not in JSON " + errorBody);
+            String[] errors = { "no error code", "no summary", "no stacktrace" };
+            return errors;
+        }
     }
 
     // Executes Query and returns results as JSONArray
diff --git a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/transaction/RecoveryIT.java b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/transaction/RecoveryIT.java
index 89cbec8..3346203 100644
--- a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/transaction/RecoveryIT.java
+++ b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/transaction/RecoveryIT.java
@@ -69,6 +69,7 @@
             }
         })[0];
         managixHomePath = new File(installerTargetPath, managixHomeDirName).getAbsolutePath();
+        LOGGER.info("MANAGIX_HOME=" + managixHomePath);
 
         String fileListPath = asterixInstallerPath.getAbsolutePath() + File.separator + "src" + File.separator + "test"
                 + File.separator + "resources" + File.separator + "transactionts" + File.separator + "data"
@@ -76,6 +77,7 @@
         String srcBasePath = asterixAppPath.getAbsolutePath();
         String destBasePath = managixHomePath + File.separator + "clusters" + File.separator + "local" + File.separator
                 + "working_dir";
+        LOGGER.info("working dir: " + destBasePath);
         prepareDataFiles(fileListPath, srcBasePath, destBasePath);
 
         pb = new ProcessBuilder();
@@ -110,7 +112,7 @@
         File outdir = new File(PATH_ACTUAL);
         FileUtils.deleteDirectory(outdir);
         TestsUtils.executeScript(pb, scriptHomePath + File.separator + "setup_teardown" + File.separator
-                + "stop_and_delete.sh");
+                + "stop.sh");
         TestsUtils.executeScript(pb, scriptHomePath + File.separator + "setup_teardown" + File.separator
                 + "shutdown.sh");
     }
diff --git a/asterix-installer/src/test/resources/transactionts/scripts/setup_teardown/stop.sh b/asterix-installer/src/test/resources/transactionts/scripts/setup_teardown/stop.sh
new file mode 100755
index 0000000..4fd44b8
--- /dev/null
+++ b/asterix-installer/src/test/resources/transactionts/scripts/setup_teardown/stop.sh
@@ -0,0 +1,2 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
index 46159ba..bba6660 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
@@ -89,14 +89,38 @@
         this.metadataLatch = new ReentrantReadWriteLock(true);
     }
 
+    public MetadataManager(IAsterixStateProxy proxy, IMetadataNode metadataNode) {
+        if (metadataNode == null) {
+            throw new Error("Null metadataNode given to MetadataManager.");
+        }
+        this.proxy = proxy;
+        this.metadataProperties = null;
+        this.metadataNode = metadataNode;
+        this.metadataLatch = new ReentrantReadWriteLock(true);
+    }
+
     @Override
-    public void init() throws RemoteException {
+    public void init() throws RemoteException, MetadataException {
         // Could be synchronized on any object. Arbitrarily chose proxy.
         synchronized (proxy) {
             if (metadataNode != null) {
                 return;
             }
-            metadataNode = proxy.getMetadataNode();
+            try {
+                int retry = 0;
+                int sleep = 64;
+                while (retry++ < 10) {
+                    metadataNode = proxy.getMetadataNode();
+                    if (metadataNode != null) {
+                        break;
+                    }
+                    System.err.println("sleeping for " + sleep + " ms");
+                    Thread.sleep(sleep);
+                    sleep *= 4;
+                }
+            } catch (InterruptedException e) {
+                throw new MetadataException(e);
+            }
             if (metadataNode == null) {
                 throw new Error("Failed to get the MetadataNode.\n" + "The MetadataNode was configured to run on NC: "
                         + metadataProperties.getMetadataNodeName());
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
index b344f69..29452f3 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
@@ -1005,9 +1005,9 @@
             try {
                 while (rangeCursor.hasNext()) {
                     rangeCursor.next();
-                    ITupleReference ref = rangeCursor.getTuple();
-                    Dataset ds = valueExtractor.getValue(jobId, rangeCursor.getTuple());
-                    datasetId = ((Dataset) valueExtractor.getValue(jobId, rangeCursor.getTuple())).getDatasetId();
+                    final ITupleReference ref = rangeCursor.getTuple();
+                    final Dataset ds = valueExtractor.getValue(jobId, ref);
+                    datasetId = ds.getDatasetId();
                     if (mostRecentDatasetId < datasetId) {
                         mostRecentDatasetId = datasetId;
                     }