after merge master
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AbstractAqlTranslator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AbstractAqlTranslator.java
index 94c074f..1d6b1c3 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AbstractAqlTranslator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AbstractAqlTranslator.java
@@ -16,6 +16,8 @@
 
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 import edu.uci.ics.asterix.aql.base.Statement;
 import edu.uci.ics.asterix.aql.expression.DatasetDecl;
@@ -30,6 +32,7 @@
 import edu.uci.ics.asterix.metadata.entities.AsterixBuiltinTypeMap;
 import edu.uci.ics.asterix.metadata.entities.Dataverse;
 import edu.uci.ics.asterix.om.types.BuiltinType;
+import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
 import edu.uci.ics.asterix.om.util.AsterixClusterProperties;
 import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
 
@@ -39,10 +42,38 @@
  */
 public abstract class AbstractAqlTranslator {
 
+    protected static final Logger LOGGER = Logger.getLogger(AbstractAqlTranslator.class.getName());
+
     protected static final Map<String, BuiltinType> builtinTypeMap = AsterixBuiltinTypeMap.getBuiltinTypes();
 
     public void validateOperation(Dataverse defaultDataverse, Statement stmt) throws AsterixException {
 
+        if (!AsterixClusterProperties.INSTANCE.getState().equals(AsterixClusterProperties.State.ACTIVE)) {
+            int maxWaitCycles = AsterixAppContextInfo.getInstance().getExternalProperties().getMaxWaitClusterActive();
+            int waitCycleCount = 0;
+            try {
+                while (!AsterixClusterProperties.INSTANCE.getState().equals(AsterixClusterProperties.State.ACTIVE)
+                        && waitCycleCount < maxWaitCycles) {
+                    Thread.sleep(1000);
+                    waitCycleCount++;
+                }
+            } catch (InterruptedException e) {
+                if (LOGGER.isLoggable(Level.WARNING)) {
+                    LOGGER.warning("Thread interrupted while waiting for cluster to be "
+                            + AsterixClusterProperties.State.ACTIVE);
+                }
+            }
+            if (!AsterixClusterProperties.INSTANCE.getState().equals(AsterixClusterProperties.State.ACTIVE)) {
+                throw new AsterixException(" Asterix Cluster is in " + AsterixClusterProperties.State.UNUSABLE
+                        + " state." + "\n One or more Node Controllers have left or haven't joined yet.\n");
+            } else {
+                if (LOGGER.isLoggable(Level.INFO)) {
+                    LOGGER.info("Cluster is now " + AsterixClusterProperties.State.ACTIVE);
+                }
+            }
+        }
+
+
         if (AsterixClusterProperties.INSTANCE.getState().equals(AsterixClusterProperties.State.UNUSABLE)) {
             throw new AsterixException(" Asterix Cluster is in " + AsterixClusterProperties.State.UNUSABLE + " state."
                     + "\n One or more Node Controllers have left.\n");
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index 96d0617..5423d2a 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -14,7 +14,6 @@
  */
 package edu.uci.ics.asterix.hyracks.bootstrap;
 
-import java.rmi.RemoteException;
 import java.rmi.server.UnicastRemoteObject;
 import java.util.HashMap;
 import java.util.Map;
@@ -56,10 +55,7 @@
         if (LOGGER.isLoggable(Level.INFO)) {
             LOGGER.info("Starting Asterix node controller  TAKE NOTE: " + nodeId);
         }
-        JVMShutdownHook sHook = new JVMShutdownHook(this);
-        Runtime.getRuntime().addShutdownHook(sHook);
 
-     
         runtimeContext = new AsterixAppRuntimeContext(ncApplicationContext);
         runtimeContext.initialize();
         ncApplicationContext.setApplicationObject(runtimeContext);
@@ -129,22 +125,32 @@
 
         isMetadataNode = nodeId.equals(metadataProperties.getMetadataNodeName());
         if (isMetadataNode) {
-            registerRemoteMetadataNode(proxy);
-
             if (LOGGER.isLoggable(Level.INFO)) {
                 LOGGER.info("Bootstrapping metadata");
             }
-            MetadataManager.INSTANCE = new MetadataManager(proxy, metadataProperties);
-            MetadataManager.INSTANCE.init();
+            MetadataNode.INSTANCE.initialize(runtimeContext);
+
+            // This is a special case, we just give the metadataNode directly.
+            // This way we can delay the registration of the metadataNode until
+            // it is completely initialized.
+            MetadataManager.INSTANCE = new MetadataManager(proxy, MetadataNode.INSTANCE);
             MetadataBootstrap.startUniverse(((IAsterixPropertiesProvider) runtimeContext), ncApplicationContext,
                     systemState == SystemState.NEW_UNIVERSE);
             MetadataBootstrap.startDDLRecovery();
+
+            IMetadataNode stub = null;
+            stub = (IMetadataNode) UnicastRemoteObject.exportObject(MetadataNode.INSTANCE, 0);
+            proxy.setMetadataNode(stub);
+
+            if (LOGGER.isLoggable(Level.INFO)) {
+                LOGGER.info("Metadata node bound");
+            }
         }
 
         if (LOGGER.isLoggable(Level.INFO)) {
             LOGGER.info("Starting lifecycle components");
         }
-        
+
         Map<String, String> lifecycleMgmtConfiguration = new HashMap<String, String>();
         String key = LifeCycleComponentManager.Config.DUMP_PATH_KEY;
         String value = metadataProperties.getCoredumpPath(nodeId);
@@ -156,7 +162,7 @@
         if (LOGGER.isLoggable(Level.INFO)) {
             LOGGER.info("Configured:" + LifeCycleComponentManager.INSTANCE);
         }
-        
+
         LifeCycleComponentManager.INSTANCE.startAll();
 
         IRecoveryManager recoveryMgr = runtimeContext.getTransactionSubsystem().getRecoveryManager();
@@ -166,40 +172,4 @@
         // reclaim storage for orphaned index artifacts in NCs.
     }
 
-    public void registerRemoteMetadataNode(IAsterixStateProxy proxy) throws RemoteException {
-        IMetadataNode stub = null;
-        MetadataNode.INSTANCE.initialize(runtimeContext);
-        stub = (IMetadataNode) UnicastRemoteObject.exportObject(MetadataNode.INSTANCE, 0);
-        proxy.setMetadataNode(stub);
-
-        if (LOGGER.isLoggable(Level.INFO)) {
-            LOGGER.info("Metadata node bound");
-        }
-    }
-
-    /**
-     * Shutdown hook that invokes {@link NCApplicationEntryPoint#stop() stop} method.
-     */
-    private static class JVMShutdownHook extends Thread {
-
-        private final NCApplicationEntryPoint ncAppEntryPoint;
-
-        public JVMShutdownHook(NCApplicationEntryPoint ncAppEntryPoint) {
-            this.ncAppEntryPoint = ncAppEntryPoint;
-        }
-
-        public void run() {
-            if (LOGGER.isLoggable(Level.INFO)) {
-                LOGGER.info("Shutdown hook in progress");
-            }
-            try {
-                ncAppEntryPoint.stop();
-            } catch (Exception e) {
-                if (LOGGER.isLoggable(Level.WARNING)) {
-                    LOGGER.warning("Exception in executing shutdown hook" + e);
-                }
-            }
-        }
-    }
-
 }
\ No newline at end of file
diff --git a/asterix-app/src/main/resources/asterix-build-configuration.xml b/asterix-app/src/main/resources/asterix-build-configuration.xml
index 7a91205..d798cd5 100644
--- a/asterix-app/src/main/resources/asterix-build-configuration.xml
+++ b/asterix-app/src/main/resources/asterix-build-configuration.xml
@@ -25,6 +25,15 @@
 		<ncId>nc2</ncId>
 		<txnLogDirPath>target/txnLogDir/nc2</txnLogDirPath>
 	</transactionLogDir>
+
+        <property>
+                <name>max.wait.active.cluster</name>
+                <value>60</value>
+                <description>Maximum wait (in seconds) for a cluster to be ACTIVE (all nodes are available)
+                        before a submitted query/statement  can be executed. (Default = 60 seconds)
+                </description>
+        </property>
+
 	<property>
 		<name>log.level</name>
 		<value>WARNING</value>
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixExternalProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixExternalProperties.java
index e975f60..674263f 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixExternalProperties.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixExternalProperties.java
@@ -27,6 +27,9 @@
     private static final String EXTERNAL_APISERVER_KEY = "api.port";
     private static int EXTERNAL_APISERVER_DEFAULT = 19002;
 
+    private static final String EXTERNAL_MAX_WAIT_FOR_ACTIVE_CLUSTER = "max.wait.active.cluster";
+    private static int EXTERNAL_MAX_WAIT_FOR_ACTIVE_CLUSTER_DEFAULT = 60;
+
     public AsterixExternalProperties(AsterixPropertiesAccessor accessor) {
         super(accessor);
     }
@@ -45,4 +48,9 @@
         return accessor.getProperty(EXTERNAL_LOGLEVEL_KEY, EXTERNAL_LOGLEVEL_DEFAULT,
                 PropertyInterpreters.getLevelPropertyInterpreter());
     }
+
+    public int getMaxWaitClusterActive() {
+        return accessor.getProperty(EXTERNAL_MAX_WAIT_FOR_ACTIVE_CLUSTER, EXTERNAL_MAX_WAIT_FOR_ACTIVE_CLUSTER_DEFAULT,
+                PropertyInterpreters.getIntegerPropertyInterpreter());
+    }
 }
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/GlobalConfig.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/GlobalConfig.java
index 21913b0..e00c432 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/GlobalConfig.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/GlobalConfig.java
@@ -14,7 +14,7 @@
  */
 package edu.uci.ics.asterix.common.config;
 
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.logging.Logger;
 
@@ -45,7 +45,7 @@
 
     public static final Map<String, String> DEFAULT_COMPACTION_POLICY_PROPERTIES;
     static {
-        DEFAULT_COMPACTION_POLICY_PROPERTIES = new HashMap<String, String>();
+        DEFAULT_COMPACTION_POLICY_PROPERTIES = new LinkedHashMap<String, String>();
         DEFAULT_COMPACTION_POLICY_PROPERTIES.put("max-mergable-component-size", "1073741824"); // 1GB
         DEFAULT_COMPACTION_POLICY_PROPERTIES.put("max-tolernace-component-count", "5"); // 5 components
     }
diff --git a/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java b/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java
index de68bef..0fb1e76 100644
--- a/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java
+++ b/asterix-common/src/test/java/edu/uci/ics/asterix/test/aql/TestsUtils.java
@@ -422,6 +422,7 @@
                             break;
                         case "txnqar": //qar represents query after recovery
                             try {
+				/*
 
                                 ////////////// <begin of temporary fix> ////////////////////////////
                                 //TODO
@@ -443,6 +444,9 @@
                                     resultStream = executeQuery(statement);
                                 } while (resultStream.toString().contains("Connection refused to host"));
                                 ////////////// <end of temporary fix> //////////////////////////////
+				*/
+                                InputStream resultStream = executeQuery(statement);
+
 
 
                                 qarFile = new File(actualPath + File.separator
diff --git a/asterix-installer/src/main/resources/conf/asterix-configuration.xml b/asterix-installer/src/main/resources/conf/asterix-configuration.xml
index cbee8f8..6f96191 100644
--- a/asterix-installer/src/main/resources/conf/asterix-configuration.xml
+++ b/asterix-installer/src/main/resources/conf/asterix-configuration.xml
@@ -27,6 +27,14 @@
 		</description>
 	</property>
 
+        <property>
+                <name>max.wait.active.cluster</name>
+                <value>60</value>
+                <description>Maximum wait (in seconds) for a cluster to be ACTIVE (all nodes are available)
+                        before a submitted query/statement can be executed. (Default = 60 seconds)
+                </description>
+        </property>
+
 	<property>
 		<name>storage.buffercache.pagesize</name>
 		<value>131072</value>
diff --git a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/transaction/RecoveryIT.java b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/transaction/RecoveryIT.java
index e1552d1..ada7eb7 100644
--- a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/transaction/RecoveryIT.java
+++ b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/transaction/RecoveryIT.java
@@ -70,6 +70,7 @@
             }
         })[0];
         managixHomePath = new File(installerTargetPath, managixHomeDirName).getAbsolutePath();
+        LOGGER.info("MANAGIX_HOME=" + managixHomePath);
 
         /*String fileListPath = asterixInstallerPath.getAbsolutePath() + File.separator + "src" + File.separator + "test"
                 + File.separator + "resources" + File.separator + "transactionts" + File.separator + "data"
@@ -77,8 +78,11 @@
         String srcBasePath = asterixAppPath.getAbsolutePath();
         String destBasePath = managixHomePath + File.separator + "clusters" + File.separator + "local" + File.separator
                 + "working_dir";
-        prepareDataFiles(fileListPath, srcBasePath, destBasePath);*/
 
+        LOGGER.info("working dir: " + destBasePath);
+        prepareDataFiles(fileListPath, srcBasePath, destBasePath);
+	*/
+	
         pb = new ProcessBuilder();
         env = pb.environment();
         env.put("MANAGIX_HOME", managixHomePath);
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
index 46159ba..c616fbf 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
@@ -71,6 +71,9 @@
  * with transaction ids of regular jobs or other metadata transactions.
  */
 public class MetadataManager implements IMetadataManager {
+    private static final int INITIAL_SLEEP_TIME = 64;
+    private static final int RETRY_MULTIPLIER = 4;
+    private static final int MAX_RETRY_COUNT = 6;
     // Set in init().
     public static MetadataManager INSTANCE;
     private final MetadataCache cache = new MetadataCache();
@@ -89,14 +92,37 @@
         this.metadataLatch = new ReentrantReadWriteLock(true);
     }
 
+    public MetadataManager(IAsterixStateProxy proxy, IMetadataNode metadataNode) {
+        if (metadataNode == null) {
+            throw new Error("Null metadataNode given to MetadataManager.");
+        }
+        this.proxy = proxy;
+        this.metadataProperties = null;
+        this.metadataNode = metadataNode;
+        this.metadataLatch = new ReentrantReadWriteLock(true);
+    }
+
     @Override
-    public void init() throws RemoteException {
+    public void init() throws RemoteException, MetadataException {
         // Could be synchronized on any object. Arbitrarily chose proxy.
         synchronized (proxy) {
             if (metadataNode != null) {
                 return;
             }
-            metadataNode = proxy.getMetadataNode();
+            try {
+                int retry = 0;
+                int sleep = INITIAL_SLEEP_TIME;
+                while (retry++ < MAX_RETRY_COUNT) {
+                    metadataNode = proxy.getMetadataNode();
+                    if (metadataNode != null) {
+                        break;
+                    }
+                    Thread.sleep(sleep);
+                    sleep *= RETRY_MULTIPLIER;
+                }
+            } catch (InterruptedException e) {
+                throw new MetadataException(e);
+            }
             if (metadataNode == null) {
                 throw new Error("Failed to get the MetadataNode.\n" + "The MetadataNode was configured to run on NC: "
                         + metadataProperties.getMetadataNodeName());
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
index b344f69..29452f3 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
@@ -1005,9 +1005,9 @@
             try {
                 while (rangeCursor.hasNext()) {
                     rangeCursor.next();
-                    ITupleReference ref = rangeCursor.getTuple();
-                    Dataset ds = valueExtractor.getValue(jobId, rangeCursor.getTuple());
-                    datasetId = ((Dataset) valueExtractor.getValue(jobId, rangeCursor.getTuple())).getDatasetId();
+                    final ITupleReference ref = rangeCursor.getTuple();
+                    final Dataset ds = valueExtractor.getValue(jobId, ref);
+                    datasetId = ds.getDatasetId();
                     if (mostRecentDatasetId < datasetId) {
                         mostRecentDatasetId = datasetId;
                     }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
index 455715d..1096da3 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -210,18 +210,7 @@
 
     public static void stopUniverse() throws HyracksDataException {
         // Close all BTree files in BufferCache.
-        for (int i = 0; i < primaryIndexes.length; i++) {
-            long resourceID = localResourceRepository
-                    .getResourceByName(primaryIndexes[i].getFile().getFile().getPath()).getResourceId();
-            indexLifecycleManager.close(resourceID);
-            indexLifecycleManager.unregister(resourceID);
-        }
-        for (int i = 0; i < secondaryIndexes.length; i++) {
-            long resourceID = localResourceRepository.getResourceByName(
-                    secondaryIndexes[i].getFile().getFile().getPath()).getResourceId();
-            indexLifecycleManager.close(resourceID);
-            indexLifecycleManager.unregister(resourceID);
-        }
+        // metadata datasets will be closed when the dataset life cycle manger is closed
     }
 
     public static void insertInitialDataverses(MetadataTransactionContext mdTxnCtx) throws Exception {
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
index 3f70e2b..6c9c7f2 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
@@ -23,6 +23,7 @@
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -175,7 +176,7 @@
                 cursor = ((AOrderedList) datasetDetailsRecord
                         .getValueByPos(MetadataRecordTypes.FEED_DETAILS_ARECORD_COMPACTION_POLICY_PROPERTIES_FIELD_INDEX))
                         .getCursor();
-                Map<String, String> compactionPolicyProperties = new HashMap<String, String>();
+                Map<String, String> compactionPolicyProperties = new LinkedHashMap<String, String>();
                 while (cursor.next()) {
                     ARecord field = (ARecord) cursor.get();
                     key = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_NAME_FIELD_INDEX))
@@ -216,7 +217,7 @@
                 cursor = ((AOrderedList) datasetDetailsRecord
                         .getValueByPos(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_COMPACTION_POLICY_PROPERTIES_FIELD_INDEX))
                         .getCursor();
-                Map<String, String> compactionPolicyProperties = new HashMap<String, String>();
+                Map<String, String> compactionPolicyProperties = new LinkedHashMap<String, String>();
                 String key;
                 String value;
                 while (cursor.next()) {
@@ -379,10 +380,8 @@
         IACursor cursor = list.getCursor();
         while (cursor.next()) {
             ARecord field = (ARecord) cursor.get();
-            key = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_NAME_FIELD_INDEX))
-                    .getStringValue();
-            value = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_VALUE_FIELD_INDEX))
-                    .getStringValue();
+            key = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_NAME_FIELD_INDEX)).getStringValue();
+            value = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_VALUE_FIELD_INDEX)).getStringValue();
             hints.put(key, value);
         }
         return hints;