diff --git a/.gitignore b/.gitignore
index d748fde..6d41006 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,4 +7,4 @@
 asterix-app/mdtest/
 asterix-app/opttest/
 build
-asterix_logs
\ No newline at end of file
+asterix_logs
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/UnnestToDataScanRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/UnnestToDataScanRule.java
index 642ef22..cfb767f 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/UnnestToDataScanRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/UnnestToDataScanRule.java
@@ -26,6 +26,8 @@
 import edu.uci.ics.asterix.metadata.declared.ExternalFeedDataSource;
 import edu.uci.ics.asterix.metadata.entities.Dataset;
 import edu.uci.ics.asterix.metadata.entities.Dataverse;
+import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
+import edu.uci.ics.asterix.metadata.feeds.BuiltinFeedPolicies;
 import edu.uci.ics.asterix.metadata.utils.DatasetUtils;
 import edu.uci.ics.asterix.om.base.AString;
 import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
@@ -111,7 +113,6 @@
                     }
                 }
                 v.add(unnest.getVariable());
-
                 DataSourceScanOperator scan = new DataSourceScanOperator(v, metadataProvider.findDataSource(asid));
                 List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
                 scanInpList.addAll(unnest.getInputs());
@@ -155,11 +156,20 @@
                 }
 
                 AqlSourceId asid = new AqlSourceId(dataverseName, datasetName);
+                String policyName = metadataProvider.getConfig().get(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY);
+                FeedPolicy policy = metadataProvider.findFeedPolicy(dataverseName, policyName);
+                if (policy == null) {
+                    policy = BuiltinFeedPolicies.getFeedPolicy(policyName);
+                    if (policy == null) {
+                        throw new AlgebricksException("Unknown feed policy:" + policyName);
+                    }
+                }
+
                 ArrayList<LogicalVariable> v = new ArrayList<LogicalVariable>();
                 v.add(unnest.getVariable());
 
                 DataSourceScanOperator scan = new DataSourceScanOperator(v, createDummyFeedDataSource(asid, dataset,
-                        metadataProvider));
+                        metadataProvider, policy));
 
                 List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
                 scanInpList.addAll(unnest.getInputs());
@@ -184,7 +194,7 @@
     }
 
     private AqlDataSource createDummyFeedDataSource(AqlSourceId aqlId, Dataset dataset,
-            AqlMetadataProvider metadataProvider) throws AlgebricksException {
+            AqlMetadataProvider metadataProvider, FeedPolicy feedPolicy) throws AlgebricksException {
         if (!aqlId.getDataverseName().equals(
                 metadataProvider.getDefaultDataverse() == null ? null : metadataProvider.getDefaultDataverse()
                         .getDataverseName())) {
@@ -194,6 +204,7 @@
         IAType itemType = metadataProvider.findType(dataset.getDataverseName(), tName);
         ExternalFeedDataSource extDataSource = new ExternalFeedDataSource(aqlId, dataset, itemType,
                 AqlDataSource.AqlDataSourceType.EXTERNAL_FEED);
+        extDataSource.getProperties().put(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY, feedPolicy);
         return extDataSource;
     }
 
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/CompiledStatements.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/CompiledStatements.java
index e40a163..0c606c5 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/CompiledStatements.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/CompiledStatements.java
@@ -363,12 +363,15 @@
     public static class CompiledBeginFeedStatement implements ICompiledDmlStatement {
         private String dataverseName;
         private String datasetName;
+        private String policyName;
         private Query query;
         private int varCounter;
 
-        public CompiledBeginFeedStatement(String dataverseName, String datasetName, Query query, int varCounter) {
+        public CompiledBeginFeedStatement(String dataverseName, String datasetName, String policyName, Query query,
+                int varCounter) {
             this.dataverseName = dataverseName;
             this.datasetName = datasetName;
+            this.policyName = policyName;
             this.query = query;
             this.varCounter = varCounter;
         }
@@ -399,6 +402,10 @@
         public Kind getKind() {
             return Kind.BEGIN_FEED;
         }
+
+        public String getPolicyName() {
+            return policyName;
+        }
     }
 
     public static class CompiledControlFeedStatement implements ICompiledDmlStatement {
diff --git a/asterix-app/.gitignore b/asterix-app/.gitignore
deleted file mode 100644
index ea8c4bf..0000000
--- a/asterix-app/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-/target
diff --git a/asterix-app/scripts/asterix/startallncs.sh b/asterix-app/scripts/asterix/startallncs.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/asterix/startcc.sh b/asterix-app/scripts/asterix/startcc.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/asterix/startnc.sh b/asterix-app/scripts/asterix/startnc.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/asterix/stopallncs.sh b/asterix-app/scripts/asterix/stopallncs.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/asterix/stopcc.sh b/asterix-app/scripts/asterix/stopcc.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/asterix/stopnc.sh b/asterix-app/scripts/asterix/stopnc.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/asterix/test.properties b/asterix-app/scripts/asterix/test.properties
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/idefix/startcc.sh b/asterix-app/scripts/idefix/startcc.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/idefix/startnc1.sh b/asterix-app/scripts/idefix/startnc1.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/idefix/startnc2.sh b/asterix-app/scripts/idefix/startnc2.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/idefix/stopallncs.sh b/asterix-app/scripts/idefix/stopallncs.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/idefix/stopcc.sh b/asterix-app/scripts/idefix/stopcc.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/rainbow/execute.sh b/asterix-app/scripts/rainbow/execute.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/rainbow/startallncs.sh b/asterix-app/scripts/rainbow/startallncs.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/rainbow/startcc.sh b/asterix-app/scripts/rainbow/startcc.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/rainbow/startnc.sh b/asterix-app/scripts/rainbow/startnc.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/rainbow/stopallncs.sh b/asterix-app/scripts/rainbow/stopallncs.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/rainbow/stopcc.sh b/asterix-app/scripts/rainbow/stopcc.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/scripts/rainbow/stopnc.sh b/asterix-app/scripts/rainbow/stopnc.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
index 655d249..355382a 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
@@ -80,11 +80,15 @@
 import edu.uci.ics.asterix.metadata.entities.Datatype;
 import edu.uci.ics.asterix.metadata.entities.Dataverse;
 import edu.uci.ics.asterix.metadata.entities.ExternalDatasetDetails;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity.FeedActivityType;
 import edu.uci.ics.asterix.metadata.entities.FeedDatasetDetails;
+import edu.uci.ics.asterix.metadata.entities.FeedDatasetDetails.FeedState;
 import edu.uci.ics.asterix.metadata.entities.Function;
 import edu.uci.ics.asterix.metadata.entities.Index;
 import edu.uci.ics.asterix.metadata.entities.InternalDatasetDetails;
 import edu.uci.ics.asterix.metadata.entities.NodeGroup;
+import edu.uci.ics.asterix.metadata.feeds.BuiltinFeedPolicies;
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.ATypeTag;
 import edu.uci.ics.asterix.om.types.IAType;
@@ -438,9 +442,11 @@
                     Map<String, String> configuration = ((FeedDetailsDecl) dd.getDatasetDetailsDecl())
                             .getConfiguration();
                     FunctionSignature signature = ((FeedDetailsDecl) dd.getDatasetDetailsDecl()).getFunctionSignature();
+
                     datasetDetails = new FeedDatasetDetails(InternalDatasetDetails.FileStructure.BTREE,
                             InternalDatasetDetails.PartitioningStrategy.HASH, partitioningExprs, partitioningExprs,
-                            ngName, adapter, configuration, signature, FeedDatasetDetails.FeedState.INACTIVE.toString());
+                            ngName, adapter, configuration, signature,
+                            FeedDatasetDetails.FeedState.INACTIVE.toString(), null, null);
                     break;
                 }
             }
@@ -1304,17 +1310,15 @@
         boolean bActiveTxn = true;
         metadataProvider.setMetadataTxnContext(mdTxnCtx);
         acquireReadLatch();
-
         try {
             BeginFeedStatement bfs = (BeginFeedStatement) stmt;
             String dataverseName = getActiveDataverseName(bfs.getDataverseName());
 
             CompiledBeginFeedStatement cbfs = new CompiledBeginFeedStatement(dataverseName, bfs.getDatasetName()
-                    .getValue(), bfs.getQuery(), bfs.getVarCounter());
+                    .getValue(), bfs.getPolicy(), bfs.getQuery(), bfs.getVarCounter());
 
-            Dataset dataset;
-            dataset = MetadataManager.INSTANCE.getDataset(metadataProvider.getMetadataTxnContext(), dataverseName, bfs
-                    .getDatasetName().getValue());
+            Dataset dataset = MetadataManager.INSTANCE.getDataset(metadataProvider.getMetadataTxnContext(),
+                    dataverseName, bfs.getDatasetName().getValue());
             if (dataset == null) {
                 throw new AsterixException("Unknown dataset :" + bfs.getDatasetName().getValue());
             }
@@ -1326,16 +1330,11 @@
             bfs.initialize(metadataProvider.getMetadataTxnContext(), dataset);
             cbfs.setQuery(bfs.getQuery());
             metadataProvider.getConfig().put(FunctionUtils.IMPORT_PRIVATE_FUNCTIONS, "" + Boolean.TRUE);
-
+            metadataProvider.getConfig().put(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY, cbfs.getPolicyName());
             JobSpecification compiled = rewriteCompileQuery(metadataProvider, bfs.getQuery(), cbfs);
-
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
             bActiveTxn = false;
-
-            if (compiled != null) {
-                runJob(hcc, compiled, true);
-            }
-
+            runJob(hcc, compiled, false);
         } catch (Exception e) {
             if (bActiveTxn) {
                 abort(e, e, mdTxnCtx);
@@ -1356,9 +1355,30 @@
         try {
             ControlFeedStatement cfs = (ControlFeedStatement) stmt;
             String dataverseName = getActiveDataverseName(cfs.getDataverseName());
+            
+            String datasetName = cfs.getDatasetName().getValue();
+            Dataset dataset = MetadataManager.INSTANCE.getDataset(metadataProvider.getMetadataTxnContext(),
+                    dataverseName, cfs.getDatasetName().getValue());
+            if (dataset == null) {
+                throw new AsterixException("Unknown dataset :" + cfs.getDatasetName().getValue() + " in dataverse "
+                        + dataverseName);
+            }
+            if (!dataset.getDatasetType().equals(DatasetType.FEED)) {
+                throw new AsterixException("Statement not applicable. Dataset " + cfs.getDatasetName().getValue()
+                        + " is not a " + DatasetType.FEED);
+            }
+
+            FeedActivity feedActivity = MetadataManager.INSTANCE.getRecentFeedActivity(mdTxnCtx, dataverseName,
+                    datasetName);
+
+            if (feedActivity == null || !FeedActivityType.FEED_BEGIN.equals(feedActivity.getFeedActivityType())) {
+                throw new AsterixException("Invalid operation. The feed is currently not " + FeedState.ACTIVE);
+            }
+
             CompiledControlFeedStatement clcfs = new CompiledControlFeedStatement(cfs.getOperationType(),
                     dataverseName, cfs.getDatasetName().getValue(), cfs.getAlterAdapterConfParams());
-            JobSpecification jobSpec = FeedOperations.buildControlFeedJobSpec(clcfs, metadataProvider);
+
+            JobSpecification jobSpec = FeedOperations.buildControlFeedJobSpec(clcfs, metadataProvider, feedActivity);
 
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
             bActiveTxn = false;
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/FeedOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/FeedOperations.java
index c7ad8bd..bf12375 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/FeedOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/FeedOperations.java
@@ -14,20 +14,22 @@
  */
 package edu.uci.ics.asterix.file;
 
-import java.io.File;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.logging.Logger;
 
 import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
-import edu.uci.ics.asterix.external.feed.lifecycle.AlterFeedMessage;
-import edu.uci.ics.asterix.external.feed.lifecycle.FeedMessage;
-import edu.uci.ics.asterix.external.feed.lifecycle.IFeedMessage;
-import edu.uci.ics.asterix.external.feed.lifecycle.IFeedMessage.MessageType;
 import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
 import edu.uci.ics.asterix.metadata.entities.Dataset;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity;
 import edu.uci.ics.asterix.metadata.entities.FeedDatasetDetails;
+import edu.uci.ics.asterix.metadata.feeds.AlterFeedMessage;
+import edu.uci.ics.asterix.metadata.feeds.FeedMessage;
+import edu.uci.ics.asterix.metadata.feeds.IFeedMessage;
+import edu.uci.ics.asterix.metadata.feeds.IFeedMessage.MessageType;
 import edu.uci.ics.asterix.translator.CompiledStatements.CompiledControlFeedStatement;
 import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
 import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
@@ -58,11 +60,12 @@
      * @throws AlgebricksException
      */
     public static JobSpecification buildControlFeedJobSpec(CompiledControlFeedStatement controlFeedStatement,
-            AqlMetadataProvider metadataProvider) throws AsterixException, AlgebricksException {
+            AqlMetadataProvider metadataProvider, FeedActivity feedActivity) throws AsterixException,
+            AlgebricksException {
         switch (controlFeedStatement.getOperationType()) {
             case ALTER:
             case END: {
-                return createSendMessageToFeedJobSpec(controlFeedStatement, metadataProvider);
+                return createSendMessageToFeedJobSpec(controlFeedStatement, metadataProvider, feedActivity);
             }
             default: {
                 throw new AsterixException("Unknown Operation Type: " + controlFeedStatement.getOperationType());
@@ -72,14 +75,10 @@
     }
 
     private static JobSpecification createSendMessageToFeedJobSpec(CompiledControlFeedStatement controlFeedStatement,
-            AqlMetadataProvider metadataProvider) throws AsterixException {
+            AqlMetadataProvider metadataProvider, FeedActivity feedActivity) throws AsterixException {
         String dataverseName = controlFeedStatement.getDataverseName() == null ? metadataProvider
                 .getDefaultDataverseName() : controlFeedStatement.getDataverseName();
         String datasetName = controlFeedStatement.getDatasetName();
-        String datasetPath = dataverseName + File.separator + datasetName;
-
-        LOGGER.info(" DATASETPATH: " + datasetPath);
-
         Dataset dataset;
         try {
             dataset = metadataProvider.findDataset(dataverseName, datasetName);
@@ -103,14 +102,16 @@
                 feedMessages.add(new FeedMessage(MessageType.STOP));
                 break;
             case ALTER:
-                feedMessages.add(new AlterFeedMessage(controlFeedStatement.getProperties()));
+                Map<String, Object> wrappedProperties = new HashMap<String, Object>();
+                wrappedProperties.putAll(controlFeedStatement.getProperties());
+                feedMessages.add(new AlterFeedMessage(wrappedProperties));
                 break;
         }
 
         try {
             Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> p = metadataProvider.buildFeedMessengerRuntime(
                     metadataProvider, spec, (FeedDatasetDetails) dataset.getDatasetDetails(), dataverseName,
-                    datasetName, feedMessages);
+                    datasetName, feedMessages, feedActivity);
             feedMessenger = p.first;
             messengerPc = p.second;
         } catch (AlgebricksException e) {
@@ -121,9 +122,7 @@
 
         NullSinkOperatorDescriptor nullSink = new NullSinkOperatorDescriptor(spec);
         AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, nullSink, messengerPc);
-
         spec.connect(new OneToOneConnectorDescriptor(spec), feedMessenger, 0, nullSink, 0);
-
         spec.addRoot(nullSink);
         return spec;
 
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
index dc201dd..97a536d 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
@@ -33,6 +33,8 @@
 import edu.uci.ics.asterix.metadata.MetadataManager;
 import edu.uci.ics.asterix.metadata.api.IAsterixStateProxy;
 import edu.uci.ics.asterix.metadata.bootstrap.AsterixStateProxy;
+import edu.uci.ics.asterix.metadata.feeds.FeedJobLifecycleListener;
+import edu.uci.ics.asterix.metadata.feeds.FeedLifecycleListener;
 import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
 import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
 import edu.uci.ics.hyracks.api.application.ICCApplicationEntryPoint;
@@ -56,14 +58,22 @@
         if (LOGGER.isLoggable(Level.INFO)) {
             LOGGER.info("Starting Asterix cluster controller");
         }
+
         appCtx.setThreadFactory(AsterixThreadFactory.INSTANCE);
-        AsterixAppContextInfo.initialize(appCtx);
+        AsterixAppContextInfo.initialize(appCtx, getNewHyracksClientConnection());
+
         proxy = AsterixStateProxy.registerRemoteObject();
         appCtx.setDistributedState(proxy);
 
         AsterixMetadataProperties metadataProperties = AsterixAppContextInfo.getInstance().getMetadataProperties();
         MetadataManager.INSTANCE = new MetadataManager(proxy, metadataProperties);
 
+        AsterixAppContextInfo.getInstance().getCCApplicationContext()
+                .addJobLifecycleListener(FeedLifecycleListener.INSTANCE);
+
+        AsterixAppContextInfo.getInstance().getCCApplicationContext()
+                .addClusterLifecycleListener(FeedLifecycleListener.INSTANCE);
+
         AsterixExternalProperties externalProperties = AsterixAppContextInfo.getInstance().getExternalProperties();
         setupWebServer(externalProperties);
         webServer.start();
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index 96d0617..069cdb6 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -28,11 +28,14 @@
 import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
 import edu.uci.ics.asterix.common.transactions.IRecoveryManager;
 import edu.uci.ics.asterix.common.transactions.IRecoveryManager.SystemState;
+import edu.uci.ics.asterix.event.schema.cluster.Cluster;
+import edu.uci.ics.asterix.event.schema.cluster.Node;
 import edu.uci.ics.asterix.metadata.MetadataManager;
 import edu.uci.ics.asterix.metadata.MetadataNode;
 import edu.uci.ics.asterix.metadata.api.IAsterixStateProxy;
 import edu.uci.ics.asterix.metadata.api.IMetadataNode;
 import edu.uci.ics.asterix.metadata.bootstrap.MetadataBootstrap;
+import edu.uci.ics.asterix.om.util.AsterixClusterProperties;
 import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceRepository;
 import edu.uci.ics.hyracks.api.application.INCApplicationContext;
 import edu.uci.ics.hyracks.api.application.INCApplicationEntryPoint;
@@ -59,7 +62,6 @@
         JVMShutdownHook sHook = new JVMShutdownHook(this);
         Runtime.getRuntime().addShutdownHook(sHook);
 
-     
         runtimeContext = new AsterixAppRuntimeContext(ncApplicationContext);
         runtimeContext.initialize();
         ncApplicationContext.setApplicationObject(runtimeContext);
@@ -113,6 +115,25 @@
         IAsterixStateProxy proxy = (IAsterixStateProxy) ncApplicationContext.getDistributedState();
         AsterixMetadataProperties metadataProperties = ((IAsterixPropertiesProvider) runtimeContext)
                 .getMetadataProperties();
+        if (!metadataProperties.getNodeNames().contains(nodeId)) {
+            metadataProperties.getNodeNames().add(nodeId);
+            Cluster cluster = AsterixClusterProperties.INSTANCE.getCluster();
+            Node self = null;
+            for (Node node : cluster.getSubstituteNodes().getNode()) {
+                if (node.getId().equalsIgnoreCase(nodeId)) {
+                    String storeDir = node.getStore() == null ? cluster.getStore() : node.getStore();
+                    metadataProperties.getStores().put(nodeId, storeDir.split(","));
+                    self = node;
+                    break;
+                }
+            }
+            if (self != null) {
+                cluster.getSubstituteNodes().getNode().remove(self);
+                cluster.getNode().add(self);
+            } else {
+                throw new IllegalStateException("Unknown node joining the cluster");
+            }
+        }
 
         if (systemState == SystemState.NEW_UNIVERSE) {
             if (LOGGER.isLoggable(Level.INFO)) {
@@ -130,12 +151,15 @@
         isMetadataNode = nodeId.equals(metadataProperties.getMetadataNodeName());
         if (isMetadataNode) {
             registerRemoteMetadataNode(proxy);
+        }
+        MetadataManager.INSTANCE = new MetadataManager(proxy, metadataProperties);
+        MetadataManager.INSTANCE.init();
 
+        if (isMetadataNode) {
             if (LOGGER.isLoggable(Level.INFO)) {
                 LOGGER.info("Bootstrapping metadata");
             }
-            MetadataManager.INSTANCE = new MetadataManager(proxy, metadataProperties);
-            MetadataManager.INSTANCE.init();
+
             MetadataBootstrap.startUniverse(((IAsterixPropertiesProvider) runtimeContext), ncApplicationContext,
                     systemState == SystemState.NEW_UNIVERSE);
             MetadataBootstrap.startDDLRecovery();
@@ -144,7 +168,7 @@
         if (LOGGER.isLoggable(Level.INFO)) {
             LOGGER.info("Starting lifecycle components");
         }
-        
+
         Map<String, String> lifecycleMgmtConfiguration = new HashMap<String, String>();
         String key = LifeCycleComponentManager.Config.DUMP_PATH_KEY;
         String value = metadataProperties.getCoredumpPath(nodeId);
@@ -156,7 +180,7 @@
         if (LOGGER.isLoggable(Level.INFO)) {
             LOGGER.info("Configured:" + LifeCycleComponentManager.INSTANCE);
         }
-        
+
         LifeCycleComponentManager.INSTANCE.startAll();
 
         IRecoveryManager recoveryMgr = runtimeContext.getTransactionSubsystem().getRecoveryManager();
diff --git a/asterix-app/src/main/resources/test.properties b/asterix-app/src/main/resources/test.properties
old mode 100755
new mode 100644
diff --git a/asterix-app/src/test/resources/runtimets/queries/distinct/query-issue443-2/query-issue443-2.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/distinct/query-issue443-2/query-issue443-2.2.update.aql
deleted file mode 100644
index e69de29..0000000
--- a/asterix-app/src/test/resources/runtimets/queries/distinct/query-issue443-2/query-issue443-2.2.update.aql
+++ /dev/null
diff --git a/asterix-app/src/test/resources/runtimets/queries/distinct/query-issue443/query-issue443.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/distinct/query-issue443/query-issue443.1.ddl.aql
deleted file mode 100644
index e69de29..0000000
--- a/asterix-app/src/test/resources/runtimets/queries/distinct/query-issue443/query-issue443.1.ddl.aql
+++ /dev/null
diff --git a/asterix-app/src/test/resources/runtimets/queries/distinct/query-issue443/query-issue443.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/distinct/query-issue443/query-issue443.2.update.aql
deleted file mode 100644
index e69de29..0000000
--- a/asterix-app/src/test/resources/runtimets/queries/distinct/query-issue443/query-issue443.2.update.aql
+++ /dev/null
diff --git a/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue442/query-issue442.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue442/query-issue442.1.ddl.aql
deleted file mode 100644
index e69de29..0000000
--- a/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue442/query-issue442.1.ddl.aql
+++ /dev/null
diff --git a/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue442/query-issue442.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue442/query-issue442.2.update.aql
deleted file mode 100644
index e69de29..0000000
--- a/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue442/query-issue442.2.update.aql
+++ /dev/null
diff --git a/asterix-app/src/test/resources/runtimets/results/open-closed/query-issue423/query-issue423.1.adm b/asterix-app/src/test/resources/runtimets/results/open-closed/query-issue423/query-issue423.1.adm
deleted file mode 100644
index e69de29..0000000
--- a/asterix-app/src/test/resources/runtimets/results/open-closed/query-issue423/query-issue423.1.adm
+++ /dev/null
diff --git a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/BeginFeedStatement.java b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/BeginFeedStatement.java
index a119e2a..97e8c3c 100644
--- a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/BeginFeedStatement.java
+++ b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/BeginFeedStatement.java
@@ -24,7 +24,6 @@
 import edu.uci.ics.asterix.aql.parser.ParseException;
 import edu.uci.ics.asterix.aql.util.FunctionUtils;
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
-import edu.uci.ics.asterix.common.functions.FunctionConstants;
 import edu.uci.ics.asterix.common.functions.FunctionSignature;
 import edu.uci.ics.asterix.metadata.MetadataException;
 import edu.uci.ics.asterix.metadata.MetadataManager;
@@ -32,17 +31,20 @@
 import edu.uci.ics.asterix.metadata.entities.Dataset;
 import edu.uci.ics.asterix.metadata.entities.FeedDatasetDetails;
 import edu.uci.ics.asterix.metadata.entities.Function;
+import edu.uci.ics.asterix.metadata.feeds.BuiltinFeedPolicies;
 
 public class BeginFeedStatement implements Statement {
 
     private final Identifier dataverseName;
     private final Identifier datasetName;
+    private final String policy;
     private Query query;
     private int varCounter;
 
-    public BeginFeedStatement(Identifier dataverseName, Identifier datasetName, int varCounter) {
+    public BeginFeedStatement(Identifier dataverseName, Identifier datasetName, String policy, int varCounter) {
         this.dataverseName = dataverseName;
         this.datasetName = datasetName;
+        this.policy = policy != null ? policy : BuiltinFeedPolicies.DEFAULT_POLICY.getPolicyName();
         this.varCounter = varCounter;
     }
 
@@ -110,6 +112,10 @@
         return Kind.BEGIN_FEED;
     }
 
+    public String getPolicy() {
+        return policy;
+    }
+
     @Override
     public <R, T> R accept(IAqlExpressionVisitor<R, T> visitor, T arg) throws AsterixException {
         return visitor.visitBeginFeedStatement(this, arg);
diff --git a/asterix-aql/src/main/javacc/AQL.jj b/asterix-aql/src/main/javacc/AQL.jj
index bd21581..cc542ff 100644
--- a/asterix-aql/src/main/javacc/AQL.jj
+++ b/asterix-aql/src/main/javacc/AQL.jj
@@ -488,6 +488,18 @@
     }
 }
 
+String GetPolicy() throws ParseException:
+{
+  String policy = null;
+}
+{
+   "using" "policy" policy = Identifier() 
+   {
+     return policy;
+   }
+   
+}   
+
 FunctionSignature FunctionSignature() throws ParseException:
 {
   Pair<Identifier,Identifier> pairId = null;
@@ -748,12 +760,13 @@
   Pair<Identifier,Identifier> nameComponents = null;
   Map<String,String> configuration = null;
   Statement stmt = null;
+  String policy = null;
 }
 {
   (
-    "begin" "feed" nameComponents = QualifiedName()
+    "begin" "feed" nameComponents = QualifiedName()  (policy = GetPolicy())? 
       {
-        stmt = new BeginFeedStatement(nameComponents.first, nameComponents.second, getVarCounter());
+        stmt = new BeginFeedStatement(nameComponents.first, nameComponents.second, policy, getVarCounter());
       }
     | "suspend" "feed" nameComponents = QualifiedName()
       {
diff --git a/asterix-common/pom.xml b/asterix-common/pom.xml
index 7d1786e..544ade9 100644
--- a/asterix-common/pom.xml
+++ b/asterix-common/pom.xml
@@ -70,6 +70,40 @@
 							<generateDirectory>${project.build.directory}/generated-sources/configuration</generateDirectory>
 						</configuration>
 					</execution>
+					<execution>
+						<id>cluster</id>
+						<goals>
+							<goal>generate</goal>
+						</goals>
+						<configuration>
+							<args>
+								<arg>-Xsetters</arg>
+								<arg>-Xvalue-constructor</arg>
+							</args>
+							<plugins>
+								<plugin>
+									<groupId>org.jvnet.jaxb2_commons</groupId>
+									<artifactId>jaxb2-basics</artifactId>
+									<version>0.6.2</version>
+								</plugin>
+								<plugin>
+									<groupId>org.jvnet.jaxb2_commons</groupId>
+									<artifactId>jaxb2-value-constructor</artifactId>
+									<version>3.0</version>
+								</plugin>
+							</plugins>
+							<schemaDirectory>src/main/resources/schema</schemaDirectory>
+							<schemaIncludes>
+								<include>cluster.xsd</include>
+							</schemaIncludes>
+							<generatePackage>edu.uci.ics.asterix.event.schema.cluster</generatePackage>
+							<generateDirectory>${project.build.directory}/generated-sources/cluster</generateDirectory>
+							<bindingDirectory>src/main/resources/schema</bindingDirectory>
+							<bindingIncludes>
+								<bindingInclude>jaxb-bindings.xjb</bindingInclude>
+							</bindingIncludes>
+						</configuration>
+					</execution>
 				</executions>
 			</plugin>
 			<plugin>
@@ -116,6 +150,14 @@
 			<artifactId>hyracks-storage-am-lsm-common</artifactId>
 		</dependency>
 		<dependency>
+			<groupId>edu.uci.ics.hyracks</groupId>
+			<artifactId>hyracks-storage-am-common</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>edu.uci.ics.hyracks</groupId>
+			<artifactId>hyracks-api</artifactId>
+		</dependency>
+		<dependency>
 			<groupId>edu.uci.ics.asterix</groupId>
 			<artifactId>asterix-test-framework</artifactId>
 			<version>0.8.1-SNAPSHOT</version>
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixExternalProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixExternalProperties.java
index e975f60..7bab09d 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixExternalProperties.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixExternalProperties.java
@@ -27,6 +27,9 @@
     private static final String EXTERNAL_APISERVER_KEY = "api.port";
     private static int EXTERNAL_APISERVER_DEFAULT = 19002;
 
+    private static final String EXTERNAL_NC_JAVA_OPTS_KEY = "nc.java.opts";
+    private static String EXTERNAL_NC_JAVA_OPTS_DEFAULT = "-Xmx1024m";
+
     public AsterixExternalProperties(AsterixPropertiesAccessor accessor) {
         super(accessor);
     }
@@ -45,4 +48,9 @@
         return accessor.getProperty(EXTERNAL_LOGLEVEL_KEY, EXTERNAL_LOGLEVEL_DEFAULT,
                 PropertyInterpreters.getLevelPropertyInterpreter());
     }
+
+    public String getNCJavaParams() {
+        return accessor.getProperty(EXTERNAL_NC_JAVA_OPTS_KEY, EXTERNAL_NC_JAVA_OPTS_DEFAULT,
+                PropertyInterpreters.getStringPropertyInterpreter());
+    }
 }
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixMetadataProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixMetadataProperties.java
index 323f5f8..73a1f46 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixMetadataProperties.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixMetadataProperties.java
@@ -23,6 +23,10 @@
         super(accessor);
     }
 
+    public String getInstanceName() {
+        return accessor.getInstanceName();
+    }
+
     public String getMetadataNodeName() {
         return accessor.getMetadataNodeName();
     }
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java
index 2bd292e..881bae2 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java
@@ -39,6 +39,7 @@
 public class AsterixPropertiesAccessor {
     private static final Logger LOGGER = Logger.getLogger(AsterixPropertiesAccessor.class.getName());
 
+    private final String instanceName;
     private final String metadataNodeName;
     private final Set<String> nodeNames;
     private final Map<String, String[]> stores;
@@ -69,6 +70,7 @@
         } catch (JAXBException e) {
             throw new AsterixException("Failed to read configuration file " + fileName);
         }
+        instanceName = asterixConfiguration.getInstanceName();
         metadataNodeName = asterixConfiguration.getMetadataNode();
         stores = new HashMap<String, String[]>();
         List<Store> configuredStores = asterixConfiguration.getStore();
@@ -137,4 +139,8 @@
                     + "'.\n See the description: \n" + p.getDescription() + "\nDefault = " + defaultValue);
         }
     }
+
+    public String getInstanceName() {
+        return instanceName;
+    }
 }
diff --git a/asterix-common/src/main/resources/schema/asterix-conf.xsd b/asterix-common/src/main/resources/schema/asterix-conf.xsd
index f461723..3033893 100644
--- a/asterix-common/src/main/resources/schema/asterix-conf.xsd
+++ b/asterix-common/src/main/resources/schema/asterix-conf.xsd
@@ -5,7 +5,8 @@
 
 	<!-- definition of simple types -->
 
-        
+
+	<xs:element name="instanceName" type="xs:string" />
 	<xs:element name="metadataNode" type="xs:string" />
 	<xs:element name="coredumpPath" type="xs:string" />
 	<xs:element name="storeDirs" type="xs:string" />
@@ -13,9 +14,9 @@
 	<xs:element name="name" type="xs:string" />
 	<xs:element name="value" type="xs:string" />
 	<xs:element name="description" type="xs:string" />
-    <xs:element name="txnLogDirPath" type="xs:string" />
-	
-	
+	<xs:element name="txnLogDirPath" type="xs:string" />
+
+
 	<!-- definition of complex elements -->
 	<xs:element name="store">
 		<xs:complexType>
@@ -34,7 +35,7 @@
 			</xs:sequence>
 		</xs:complexType>
 	</xs:element>
-	
+
 	<xs:element name="transactionLogDir">
 		<xs:complexType>
 			<xs:sequence>
@@ -58,10 +59,11 @@
 	<xs:element name="asterixConfiguration">
 		<xs:complexType>
 			<xs:sequence>
-				<xs:element ref="mg:metadataNode" minOccurs="0"/>
+				<xs:element ref="mg:instanceName" minOccurs="0" />
+				<xs:element ref="mg:metadataNode" minOccurs="0" />
 				<xs:element ref="mg:store" maxOccurs="unbounded" />
 				<xs:element ref="mg:coredump" maxOccurs="unbounded" />
-				<xs:element ref="mg:transactionLogDir" maxOccurs="unbounded"/>
+				<xs:element ref="mg:transactionLogDir" maxOccurs="unbounded" />
 				<xs:element ref="mg:property" minOccurs="0" maxOccurs="unbounded" />
 			</xs:sequence>
 		</xs:complexType>
diff --git a/asterix-events/src/main/resources/schema/cluster.xsd b/asterix-common/src/main/resources/schema/cluster.xsd
similarity index 93%
rename from asterix-events/src/main/resources/schema/cluster.xsd
rename to asterix-common/src/main/resources/schema/cluster.xsd
index f0d5bd9..09b71d5 100644
--- a/asterix-events/src/main/resources/schema/cluster.xsd
+++ b/asterix-common/src/main/resources/schema/cluster.xsd
@@ -78,6 +78,14 @@
 		</xs:complexType>
 	</xs:element>
 
+	<xs:element name="substitute_nodes">
+		<xs:complexType>
+			<xs:sequence>
+				<xs:element ref="cl:node" maxOccurs="unbounded" />
+			</xs:sequence>
+		</xs:complexType>
+	</xs:element>
+
 	<xs:element name="cluster">
 		<xs:complexType>
 			<xs:sequence>
@@ -92,6 +100,7 @@
 				<xs:element ref="cl:working_dir" />
 				<xs:element ref="cl:master_node" />
 				<xs:element ref="cl:node" maxOccurs="unbounded" />
+				<xs:element ref="cl:substitute_nodes"/>
 			</xs:sequence>
 		</xs:complexType>
 	</xs:element>
diff --git a/asterix-events/pom.xml b/asterix-events/pom.xml
index f221df2..8457198 100644
--- a/asterix-events/pom.xml
+++ b/asterix-events/pom.xml
@@ -1,196 +1,207 @@
-<!--
- ! Copyright 2009-2013 by The Regents of the University of California
- ! Licensed under the Apache License, Version 2.0 (the "License");
- ! you may not use this file except in compliance with the License.
- ! you may obtain a copy of the License from
- ! 
- !     http://www.apache.org/licenses/LICENSE-2.0
- ! 
- ! Unless required by applicable law or agreed to in writing, software
- ! distributed under the License is distributed on an "AS IS" BASIS,
- ! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ! See the License for the specific language governing permissions and
- ! limitations under the License.
- !-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <parent>
-          <artifactId>asterix</artifactId>
-          <groupId>edu.uci.ics.asterix</groupId>
-          <version>0.8.1-SNAPSHOT</version>
-  </parent>
-  <groupId>edu.uci.ics.asterix</groupId>
-  <artifactId>asterix-events</artifactId>
-  <version>0.8.1-SNAPSHOT</version>
+<!-- ! Copyright 2009-2013 by The Regents of the University of California 
+	! Licensed under the Apache License, Version 2.0 (the "License"); ! you may 
+	not use this file except in compliance with the License. ! you may obtain 
+	a copy of the License from ! ! http://www.apache.org/licenses/LICENSE-2.0 
+	! ! Unless required by applicable law or agreed to in writing, software ! 
+	distributed under the License is distributed on an "AS IS" BASIS, ! WITHOUT 
+	WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ! See the 
+	License for the specific language governing permissions and ! limitations 
+	under the License. ! -->
 
-  <properties>
-    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-  </properties>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<artifactId>asterix</artifactId>
+		<groupId>edu.uci.ics.asterix</groupId>
+		<version>0.8.1-SNAPSHOT</version>
+	</parent>
+	<groupId>edu.uci.ics.asterix</groupId>
+	<artifactId>asterix-events</artifactId>
+	<version>0.8.1-SNAPSHOT</version>
 
-  <build>
-    <plugins>
-       <plugin>
-          <groupId>org.apache.maven.plugins</groupId>
-          <artifactId>maven-compiler-plugin</artifactId>
-          <version>2.0.2</version>
-          <configuration>
-             <source>1.6</source>
-             <target>1.6</target>
-          </configuration>
-       </plugin>
-       <plugin>
-          <groupId>org.jvnet.jaxb2.maven2</groupId>
-          <artifactId>maven-jaxb2-plugin</artifactId>
-          <executions>
-            <execution>
-              <id>event</id>
-              <goals>
-                <goal>generate</goal>
-              </goals>
-              <configuration>
-                <args>
-                  <arg>-Xsetters</arg>
-                  <arg>-Xvalue-constructor</arg>
-                </args>
-                <plugins>
-                  <plugin>
-                    <groupId>org.jvnet.jaxb2_commons</groupId>
-                    <artifactId>jaxb2-basics</artifactId>
-                    <version>0.6.2</version>
-                  </plugin>
-                  <plugin>
-                    <groupId>org.jvnet.jaxb2_commons</groupId>
-                    <artifactId>jaxb2-value-constructor</artifactId>
-                    <version>3.0</version>
-                  </plugin>
-                </plugins>
-                <schemaDirectory>src/main/resources/schema</schemaDirectory>
-                <schemaIncludes>
-	          <include>event.xsd</include>
-	        </schemaIncludes>
-                <generatePackage>edu.uci.ics.asterix.event.schema.event</generatePackage>
-                <generateDirectory>${project.build.directory}/generated-sources/event</generateDirectory>
-              </configuration>
-            </execution>
-            <execution>
-              <id>pattern</id>
-              <goals>
-                <goal>generate</goal>
-              </goals>
-              <configuration>
-                <args>
-                  <arg>-Xsetters</arg>
-                  <arg>-Xvalue-constructor</arg>
-                </args>
-                <plugins>
-                  <plugin>
-                    <groupId>org.jvnet.jaxb2_commons</groupId>
-                    <artifactId>jaxb2-basics</artifactId>
-                    <version>0.6.2</version>
-                  </plugin>
-                  <plugin>
-                    <groupId>org.jvnet.jaxb2_commons</groupId>
-                    <artifactId>jaxb2-value-constructor</artifactId>
-                    <version>3.0</version>
-                  </plugin>
-                </plugins>
-                <schemaDirectory>src/main/resources/schema</schemaDirectory>
-                <schemaIncludes>
-	          <include>pattern.xsd</include>
-	        </schemaIncludes>
-                <generatePackage>edu.uci.ics.asterix.event.schema.pattern</generatePackage>
-                <generateDirectory>${project.build.directory}/generated-sources/pattern</generateDirectory>
-              </configuration>
-            </execution>
-            <execution>
-              <id>cluster</id>
-              <goals>
-                <goal>generate</goal>
-              </goals>
-              <configuration>
-                <args>
-                  <arg>-Xsetters</arg>
-                  <arg>-Xvalue-constructor</arg>
-                </args>
-                <plugins>
-                  <plugin>
-                    <groupId>org.jvnet.jaxb2_commons</groupId>
-                    <artifactId>jaxb2-basics</artifactId>
-                    <version>0.6.2</version>
-                  </plugin>
-                  <plugin>
-                    <groupId>org.jvnet.jaxb2_commons</groupId>
-                    <artifactId>jaxb2-value-constructor</artifactId>
-                    <version>3.0</version>
-                  </plugin>
-                </plugins>
-                <schemaDirectory>src/main/resources/schema</schemaDirectory>
-                <schemaIncludes>
-	          <include>cluster.xsd</include>
-	        </schemaIncludes>
-                <generatePackage>edu.uci.ics.asterix.event.schema.cluster</generatePackage>
-                <generateDirectory>${project.build.directory}/generated-sources/cluster</generateDirectory>
-                <bindingDirectory>src/main/resources/schema</bindingDirectory>
-                <bindingIncludes>
-                  <bindingInclude>jaxb-bindings.xjb</bindingInclude>
-                </bindingIncludes>
-              </configuration>
-            </execution>
-          </executions>
-       </plugin>
-       <plugin>
-         <artifactId>maven-assembly-plugin</artifactId>
-         <version>2.2-beta-2</version>
-         <executions>
-           <execution>
-             <configuration>
-               <descriptor>src/main/assembly/binary-assembly.xml</descriptor>
-             </configuration>
-             <phase>package</phase>
-             <goals>
-               <goal>attached</goal>
-             </goals>
-           </execution>
-         </executions>
-       </plugin>
-    </plugins>
-  </build>
+	<properties>
+		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+	</properties>
 
-  <dependencies>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <version>4.8.1</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>args4j</groupId>
-      <artifactId>args4j</artifactId>
-      <version>2.0.12</version>
-      <type>jar</type>
-      <scope>compile</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.jvnet.jaxb2_commons</groupId>
-      <artifactId>jaxb2-value-constructor</artifactId>
-      <version>3.0</version>
-    </dependency>
-    <dependency>
-      <groupId>commons-io</groupId>
-      <artifactId>commons-io</artifactId>
-      <version>1.4</version>
-    </dependency>
-   <dependency>
-     <groupId>edu.uci.ics.asterix</groupId>
-     <artifactId>asterix-common</artifactId>
-     <version>0.8.1-SNAPSHOT</version>
-     <type>jar</type>
-     <scope>compile</scope>
-    </dependency>
-   <dependency>
-   	<groupId>log4j</groupId>
-   	<artifactId>log4j</artifactId>
-   	<version>1.2.9</version>
-   </dependency>
-  </dependencies>
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<version>2.0.2</version>
+				<configuration>
+					<source>1.6</source>
+					<target>1.6</target>
+				</configuration>
+			</plugin>
+			<plugin>
+				<groupId>org.jvnet.jaxb2.maven2</groupId>
+				<artifactId>maven-jaxb2-plugin</artifactId>
+				<executions>
+					<execution>
+						<id>event</id>
+						<goals>
+							<goal>generate</goal>
+						</goals>
+						<configuration>
+							<args>
+								<arg>-Xsetters</arg>
+								<arg>-Xvalue-constructor</arg>
+							</args>
+							<plugins>
+								<plugin>
+									<groupId>org.jvnet.jaxb2_commons</groupId>
+									<artifactId>jaxb2-basics</artifactId>
+									<version>0.6.2</version>
+								</plugin>
+								<plugin>
+									<groupId>org.jvnet.jaxb2_commons</groupId>
+									<artifactId>jaxb2-value-constructor</artifactId>
+									<version>3.0</version>
+								</plugin>
+							</plugins>
+							<schemaDirectory>src/main/resources/schema</schemaDirectory>
+							<schemaIncludes>
+								<include>event.xsd</include>
+							</schemaIncludes>
+							<generatePackage>edu.uci.ics.asterix.event.schema.event</generatePackage>
+							<generateDirectory>${project.build.directory}/generated-sources/event</generateDirectory>
+						</configuration>
+					</execution>
+					<execution>
+						<id>configuration</id>
+						<goals>
+							<goal>generate</goal>
+						</goals>
+						<configuration>
+							<args>
+								<arg>-Xsetters</arg>
+								<arg>-Xvalue-constructor</arg>
+							</args>
+							<plugins>
+								<plugin>
+									<groupId>org.jvnet.jaxb2_commons</groupId>
+									<artifactId>jaxb2-basics</artifactId>
+									<version>0.6.2</version>
+								</plugin>
+								<plugin>
+									<groupId>org.jvnet.jaxb2_commons</groupId>
+									<artifactId>jaxb2-value-constructor</artifactId>
+									<version>3.0</version>
+								</plugin>
+							</plugins>
+							<schemaDirectory>src/main/resources/schema</schemaDirectory>
+							<schemaIncludes>
+								<include>installer-conf.xsd</include>
+							</schemaIncludes>
+							<generatePackage>edu.uci.ics.asterix.installer.schema.conf</generatePackage>
+							<generateDirectory>${project.build.directory}/generated-sources/configuration</generateDirectory>
+						</configuration>
+					</execution>
+					<execution>
+						<id>pattern</id>
+						<goals>
+							<goal>generate</goal>
+						</goals>
+						<configuration>
+							<args>
+								<arg>-Xsetters</arg>
+								<arg>-Xvalue-constructor</arg>
+							</args>
+							<plugins>
+								<plugin>
+									<groupId>org.jvnet.jaxb2_commons</groupId>
+									<artifactId>jaxb2-basics</artifactId>
+									<version>0.6.2</version>
+								</plugin>
+								<plugin>
+									<groupId>org.jvnet.jaxb2_commons</groupId>
+									<artifactId>jaxb2-value-constructor</artifactId>
+									<version>3.0</version>
+								</plugin>
+							</plugins>
+							<schemaDirectory>src/main/resources/schema</schemaDirectory>
+							<schemaIncludes>
+								<include>pattern.xsd</include>
+							</schemaIncludes>
+							<generatePackage>edu.uci.ics.asterix.event.schema.pattern</generatePackage>
+							<generateDirectory>${project.build.directory}/generated-sources/pattern</generateDirectory>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+			<plugin>
+				<artifactId>maven-assembly-plugin</artifactId>
+				<version>2.2-beta-2</version>
+				<executions>
+					<execution>
+						<configuration>
+							<descriptor>src/main/assembly/binary-assembly.xml</descriptor>
+						</configuration>
+						<phase>package</phase>
+						<goals>
+							<goal>attached</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+	<dependencies>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<version>4.8.1</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>args4j</groupId>
+			<artifactId>args4j</artifactId>
+			<version>2.0.12</version>
+			<type>jar</type>
+			<scope>compile</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.jvnet.jaxb2_commons</groupId>
+			<artifactId>jaxb2-value-constructor</artifactId>
+			<version>3.0</version>
+		</dependency>
+		<dependency>
+			<groupId>commons-io</groupId>
+			<artifactId>commons-io</artifactId>
+			<version>1.4</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.zookeeper</groupId>
+			<artifactId>zookeeper</artifactId>
+			<version>3.4.5</version>
+			<exclusions>
+				<exclusion>
+					<groupId>com.sun.jmx</groupId>
+					<artifactId>jmxri</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>com.sun.jdmk</groupId>
+					<artifactId>jmxtools</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>javax.jms</groupId>
+					<artifactId>jms</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>edu.uci.ics.asterix</groupId>
+			<artifactId>asterix-common</artifactId>
+			<version>0.8.1-SNAPSHOT</version>
+			<type>jar</type>
+			<scope>compile</scope>
+		</dependency>
+		<dependency>
+			<groupId>log4j</groupId>
+			<artifactId>log4j</artifactId>
+			<version>1.2.9</version>
+		</dependency>
+	</dependencies>
 </project>
diff --git a/asterix-events/src/main/java/edu/uci/ics/asterix/event/driver/EventDriver.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/driver/EventDriver.java
index 641b37b..4f59855 100644
--- a/asterix-events/src/main/java/edu/uci/ics/asterix/event/driver/EventDriver.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/driver/EventDriver.java
@@ -29,7 +29,7 @@
 
 import edu.uci.ics.asterix.event.management.DefaultOutputHandler;
 import edu.uci.ics.asterix.event.management.EventUtil;
-import edu.uci.ics.asterix.event.management.EventrixClient;
+import edu.uci.ics.asterix.event.management.AsterixEventServiceClient;
 import edu.uci.ics.asterix.event.management.IOutputHandler;
 import edu.uci.ics.asterix.event.management.Randomizer;
 import edu.uci.ics.asterix.event.schema.cluster.Cluster;
@@ -87,15 +87,6 @@
 
     }
 
-    public static EventrixClient getClient(String eventsDir, Cluster cluster, boolean dryRun) throws Exception {
-        return new EventrixClient(eventsDir, cluster, dryRun, new DefaultOutputHandler());
-    }
-
-    public static EventrixClient getClient(String eventsDir, Cluster cluster, boolean dryRun,
-            IOutputHandler outputHandler) throws Exception {
-        return new EventrixClient(eventsDir, cluster, dryRun, outputHandler);
-    }
-
     public static void main(String[] args) throws Exception {
         String eventsHome = System.getenv("EVENT_HOME");
         if (eventsHome == null) {
@@ -119,9 +110,9 @@
             if (!eventConfig.dryRun) {
                 prepare(cluster);
             }
-            EventrixClient client = new EventrixClient(eventsDir, cluster, eventConfig.dryRun,
-                    new DefaultOutputHandler());
-            client.submit(patterns);
+            //AsterixEventServiceClient client = new AsterixEventServiceClient(eventsDir, cluster, eventConfig.dryRun,
+            //      new DefaultOutputHandler());
+            // client.submit(patterns);
             if (!eventConfig.dryRun) {
                 cleanup(cluster);
             }
diff --git a/asterix-events/src/main/java/edu/uci/ics/asterix/event/xml/PatternParser.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/error/EventException.java
similarity index 74%
rename from asterix-events/src/main/java/edu/uci/ics/asterix/event/xml/PatternParser.java
rename to asterix-events/src/main/java/edu/uci/ics/asterix/event/error/EventException.java
index af46e63..83ed5f4 100644
--- a/asterix-events/src/main/java/edu/uci/ics/asterix/event/xml/PatternParser.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/error/EventException.java
@@ -12,12 +12,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.event.xml;
+package edu.uci.ics.asterix.event.error;
 
-public class PatternParser {
+public class EventException extends Exception {
 
-	public static void parsePattern(String path){
-		
-	}
+    private static final long serialVersionUID = 1L;
+
+    public EventException(String message) {
+        super(message);
+    }
+
 }
-
diff --git a/asterix-events/src/main/java/edu/uci/ics/asterix/event/error/OutputHandler.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/error/OutputHandler.java
new file mode 100644
index 0000000..3bc795a
--- /dev/null
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/error/OutputHandler.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.event.error;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.Properties;
+
+import edu.uci.ics.asterix.event.management.IOutputHandler;
+import edu.uci.ics.asterix.event.management.OutputAnalysis;
+import edu.uci.ics.asterix.event.model.EventList.EventType;
+import edu.uci.ics.asterix.event.schema.pattern.Event;
+
+public class OutputHandler implements IOutputHandler {
+
+    public static IOutputHandler INSTANCE = new OutputHandler();
+
+    private OutputHandler() {
+
+    }
+
+    public OutputAnalysis reportEventOutput(Event event, String output) {
+
+        EventType eventType = EventType.valueOf(event.getType().toUpperCase());
+        boolean ignore = true;
+        String trimmedOutput = output.trim();
+        StringBuffer errorMessage = new StringBuffer();
+        switch (eventType) {
+            case FILE_TRANSFER:
+                if (trimmedOutput.length() > 0) {
+                    if (output.contains("Permission denied") || output.contains("cannot find or open")) {
+                        ignore = false;
+                        break;
+                    }
+                }
+                break;
+
+            case BACKUP:
+            case RESTORE:
+                if (trimmedOutput.length() > 0) {
+                    if (trimmedOutput.contains("AccessControlException")) {
+                        errorMessage.append("Insufficient permissions on back up directory");
+                        ignore = false;
+                    }
+                    if (output.contains("does not exist") || output.contains("File exist")
+                            || (output.contains("No such file or directory"))) {
+                        ignore = true;
+                    } else {
+                        ignore = false;
+                    }
+                }
+                break;
+
+            case NODE_INFO:
+                Properties p = new Properties();
+                try {
+                    p.load(new ByteArrayInputStream(trimmedOutput.getBytes()));
+                } catch (IOException e) {
+                }
+                String javaVersion = (String) p.get("java_version");
+                if (p.get("java_version") == null) {
+                    errorMessage.append("Java not installed on " + event.getNodeid().getValue().getAbsvalue());
+                    ignore = false;
+                } else if (!javaVersion.contains("1.7")) {
+                    errorMessage.append("Asterix requires Java 1.7.x. Incompatible version found on  "
+                            + event.getNodeid().getValue().getAbsvalue() + "\n");
+                    ignore = false;
+                }
+                break;
+        }
+        if (ignore) {
+            return new OutputAnalysis(true, null);
+        } else {
+            return new OutputAnalysis(false, errorMessage.toString());
+        }
+    }
+}
diff --git a/asterix-events/src/main/java/edu/uci/ics/asterix/event/error/VerificationUtil.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/error/VerificationUtil.java
new file mode 100644
index 0000000..5fcc5cf
--- /dev/null
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/error/VerificationUtil.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.event.error;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.asterix.event.model.AsterixInstance;
+import edu.uci.ics.asterix.event.model.AsterixInstance.State;
+import edu.uci.ics.asterix.event.model.AsterixRuntimeState;
+import edu.uci.ics.asterix.event.model.ProcessInfo;
+import edu.uci.ics.asterix.event.schema.cluster.Cluster;
+import edu.uci.ics.asterix.event.schema.cluster.Node;
+import edu.uci.ics.asterix.event.service.AsterixEventService;
+import edu.uci.ics.asterix.event.service.AsterixEventServiceUtil;
+
+public class VerificationUtil {
+
+    private static final String VERIFY_SCRIPT_PATH = AsterixEventService.getEventHome() + File.separator + "scripts" + File.separator + "verify.sh";
+
+    public static AsterixRuntimeState getAsterixRuntimeState(AsterixInstance instance) throws Exception {
+
+        Cluster cluster = instance.getCluster();
+        List<String> args = new ArrayList<String>();
+        args.add(instance.getName());
+        args.add(instance.getCluster().getMasterNode().getClusterIp());
+        for (Node node : cluster.getNode()) {
+            args.add(node.getClusterIp());
+            args.add(instance.getName() + "_" + node.getId());
+        }
+        Thread.sleep(2000);
+        String output = AsterixEventServiceUtil.executeLocalScript(VERIFY_SCRIPT_PATH, args);
+        boolean ccRunning = true;
+        List<String> failedNCs = new ArrayList<String>();
+        String[] infoFields;
+        ProcessInfo pInfo;
+        List<ProcessInfo> processes = new ArrayList<ProcessInfo>();
+
+        for (String line : output.split("\n")) {
+            String nodeid = null;
+            infoFields = line.split(":");
+            try {
+                int pid = Integer.parseInt(infoFields[3]);
+                if (infoFields[0].equals("NC")) {
+                    nodeid = infoFields[2].split("_")[1];
+                } else {
+                    nodeid = instance.getCluster().getMasterNode().getId();
+                }
+                pInfo = new ProcessInfo(infoFields[0], infoFields[1], nodeid, pid);
+                processes.add(pInfo);
+            } catch (Exception e) {
+                if (infoFields[0].equalsIgnoreCase("CC")) {
+                    ccRunning = false;
+                } else {
+                    failedNCs.add(infoFields[1]);
+                }
+            }
+        }
+        return new AsterixRuntimeState(processes, failedNCs, ccRunning);
+    }
+
+    public static void updateInstanceWithRuntimeDescription(AsterixInstance instance, AsterixRuntimeState state,
+            boolean expectedRunning) {
+        StringBuffer summary = new StringBuffer();
+        if (expectedRunning) {
+            if (!state.isCcRunning()) {
+                summary.append("Cluster Controller not running at " + instance.getCluster().getMasterNode().getId()
+                        + "\n");
+                instance.setState(State.UNUSABLE);
+            }
+            if (state.getFailedNCs() != null && !state.getFailedNCs().isEmpty()) {
+                summary.append("Node Controller not running at the following nodes" + "\n");
+                for (String failedNC : state.getFailedNCs()) {
+                    summary.append(failedNC + "\n");
+                }
+                instance.setState(State.UNUSABLE);
+            }
+            if (!(instance.getState().equals(State.UNUSABLE))) {
+                instance.setState(State.ACTIVE);
+            }
+        } else {
+            if (state.getProcesses() != null && state.getProcesses().size() > 0) {
+                summary.append("Following process still running " + "\n");
+                for (ProcessInfo pInfo : state.getProcesses()) {
+                    summary.append(pInfo + "\n");
+                }
+                instance.setState(State.UNUSABLE);
+            } else {
+                instance.setState(State.INACTIVE);
+            }
+        }
+        state.setSummary(summary.toString());
+        instance.setAsterixRuntimeStates(state);
+    }
+
+    public static void verifyBackupRestoreConfiguration(String hdfsUrl, String hadoopVersion, String hdfsBackupDir)
+            throws Exception {
+        StringBuffer errorCheck = new StringBuffer();
+        if (hdfsUrl == null || hdfsUrl.length() == 0) {
+            errorCheck.append("\n HDFS Url not configured");
+        }
+        if (hadoopVersion == null || hadoopVersion.length() == 0) {
+            errorCheck.append("\n HDFS version not configured");
+        }
+        if (hdfsBackupDir == null || hdfsBackupDir.length() == 0) {
+            errorCheck.append("\n HDFS backup directory not configured");
+        }
+        if (errorCheck.length() > 0) {
+            throw new Exception("Incomplete hdfs configuration" + errorCheck);
+        }
+    }
+}
diff --git a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventrixClient.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/AsterixEventServiceClient.java
similarity index 65%
rename from asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventrixClient.java
rename to asterix-events/src/main/java/edu/uci/ics/asterix/event/management/AsterixEventServiceClient.java
index 03f4061..5054aa4 100644
--- a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventrixClient.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/AsterixEventServiceClient.java
@@ -16,6 +16,7 @@
 
 import java.io.File;
 import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -24,6 +25,7 @@
 
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
 import javax.xml.bind.Unmarshaller;
 
 import edu.uci.ics.asterix.event.driver.EventDriver;
@@ -35,10 +37,12 @@
 import edu.uci.ics.asterix.event.schema.pattern.Pattern;
 import edu.uci.ics.asterix.event.schema.pattern.Patterns;
 import edu.uci.ics.asterix.event.schema.pattern.Value;
+import edu.uci.ics.asterix.event.service.AsterixEventServiceUtil;
+import edu.uci.ics.asterix.installer.schema.conf.Configuration;
 
-public class EventrixClient {
+public class AsterixEventServiceClient {
 
-    private static final Logger LOGGER = Logger.getLogger(EventrixClient.class.getName());
+    private static final Logger LOGGER = Logger.getLogger(AsterixEventServiceClient.class.getName());
 
     private EventTask[] tasks;
     private boolean dryRun = false;
@@ -48,21 +52,26 @@
     private IPatternListener listener;
     private IOutputHandler outputHandler;
     private Events events;
-    private String eventsDir;
+    private String eventsHomeDir;
+    private Configuration configuration;
 
-    public EventrixClient(String eventsDir, Cluster cluster, boolean dryRun, IOutputHandler outputHandler)
-            throws Exception {
-        this.eventsDir = eventsDir;
+    public AsterixEventServiceClient(Configuration configuration, String eventsHomeDir, Cluster cluster,
+            boolean transferArtifacts, boolean dryRun, IOutputHandler outputHandler) throws Exception {
+        this.eventsHomeDir = eventsHomeDir;
         this.events = initializeEvents();
         this.cluster = cluster;
         this.dryRun = dryRun;
+        this.configuration = configuration;
         this.outputHandler = outputHandler;
-        if (!dryRun) {
-            initializeCluster(eventsDir);
+        if (!dryRun && transferArtifacts) {
+            initializeCluster(getEventsDir());
         }
     }
 
     public void submit(Patterns patterns) throws Exception {
+        if (patterns.getPattern().isEmpty()) {
+            return;
+        }
         initTasks(patterns);
         try {
             waitForCompletion();
@@ -103,7 +112,7 @@
     }
 
     public String getEventsDir() {
-        return eventsDir;
+        return eventsHomeDir + File.separator + AsterixEventServiceUtil.EVENT_DIR;
     }
 
     public synchronized void notifyCompletion(EventTaskReport report) {
@@ -156,25 +165,34 @@
         submit(patterns);
     }
 
-	private Patterns initPattern(String eventsDir) {
-		Nodeid nodeid = new Nodeid(new Value(null,
-				EventDriver.CLIENT_NODE.getId()));
-		List<Pattern> patternList = new ArrayList<Pattern>();
-		String workingDir = cluster.getWorkingDir().getDir();
-		String username = cluster.getUsername() == null ? System
-				.getProperty("user.name") : cluster.getUsername();
-		patternList.add(getDirectoryTransferPattern(username, eventsDir,
-				nodeid, cluster.getMasterNode().getClusterIp(), workingDir));
+    private Patterns initPattern(String eventsDir) throws Exception {
+        Nodeid nodeid = new Nodeid(new Value(null, EventDriver.CLIENT_NODE.getId()));
+        List<Pattern> patternList = new ArrayList<Pattern>();
+        String workingDir = cluster.getWorkingDir().getDir();
+        String username = cluster.getUsername() == null ? System.getProperty("user.name") : cluster.getUsername();
+        patternList.add(getDirectoryTransferPattern(username, eventsDir, nodeid,
+                cluster.getMasterNode().getClusterIp(), workingDir));
 
-		if (!cluster.getWorkingDir().isNFS()) {
-			for (Node node : cluster.getNode()) {
-				patternList.add(getDirectoryTransferPattern(username,
-						eventsDir, nodeid, node.getClusterIp(), workingDir));
-			}
-		}
-		Patterns patterns = new Patterns(patternList);
-		return patterns;
-	}
+        JAXBContext ctx = JAXBContext.newInstance(Configuration.class);
+        Marshaller marshaller = ctx.createMarshaller();
+        marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+        String outputPath = System.getProperty("user.dir") + File.separator + "conf" + File.separator
+                + "configuration.xml";
+        marshaller.marshal(configuration, new FileOutputStream(outputPath));
+
+        patternList.add(getFileTransferPattern(username, outputPath, nodeid, cluster.getMasterNode().getClusterIp(),
+                workingDir));
+
+        if (!cluster.getWorkingDir().isNFS()) {
+            for (Node node : cluster.getNode()) {
+                patternList.add(getDirectoryTransferPattern(username, eventsDir, nodeid, node.getClusterIp(),
+                        workingDir));
+
+            }
+        }
+        Patterns patterns = new Patterns(patternList);
+        return patterns;
+    }
 
     private Pattern getDirectoryTransferPattern(String username, String src, Nodeid srcNode, String destNodeIp,
             String destDir) {
@@ -183,16 +201,27 @@
         return new Pattern(null, 1, null, event);
     }
 
+    private Pattern getFileTransferPattern(String username, String src, Nodeid srcNode, String destNodeIp,
+            String destDir) {
+        String pargs = username + " " + src + " " + destNodeIp + " " + destDir;
+        Event event = new Event("file_transfer", srcNode, pargs);
+        return new Pattern(null, 1, null, event);
+    }
+
     public IOutputHandler getErrorHandler() {
         return outputHandler;
     }
 
     private Events initializeEvents() throws JAXBException, FileNotFoundException {
-        File file = new File(eventsDir + File.separator + "events" + File.separator + "events.xml");
+        File file = new File(getEventsDir() + File.separator + "events.xml");
         JAXBContext eventCtx = JAXBContext.newInstance(Events.class);
         Unmarshaller unmarshaller = eventCtx.createUnmarshaller();
         events = (Events) unmarshaller.unmarshal(file);
         return events;
     }
 
+    public String getEventsHomeDir() {
+        return eventsHomeDir;
+    }
+
 }
diff --git a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/DefaultOutputHandler.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/DefaultOutputHandler.java
index d05c5ab..f7155f3 100644
--- a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/DefaultOutputHandler.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/DefaultOutputHandler.java
@@ -16,6 +16,7 @@
 
 import edu.uci.ics.asterix.event.schema.pattern.Event;
 
+
 public class DefaultOutputHandler implements IOutputHandler {
 
     @Override
diff --git a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventExecutor.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventExecutor.java
index 45499f4..fd877e6 100644
--- a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventExecutor.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventExecutor.java
@@ -23,11 +23,13 @@
 
 import org.apache.commons.io.IOUtils;
 
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
 import edu.uci.ics.asterix.event.driver.EventDriver;
 import edu.uci.ics.asterix.event.schema.cluster.Cluster;
 import edu.uci.ics.asterix.event.schema.cluster.Node;
 import edu.uci.ics.asterix.event.schema.cluster.Property;
 import edu.uci.ics.asterix.event.schema.pattern.Pattern;
+import edu.uci.ics.asterix.event.service.AsterixEventServiceUtil;
 
 public class EventExecutor {
 
@@ -40,10 +42,11 @@
     private static final String DAEMON = "DAEMON";
 
     public void executeEvent(Node node, String script, List<String> args, boolean isDaemon, Cluster cluster,
-            Pattern pattern, IOutputHandler outputHandler, EventrixClient client) throws IOException {
+            Pattern pattern, IOutputHandler outputHandler, AsterixEventServiceClient client) throws IOException {
         List<String> pargs = new ArrayList<String>();
         pargs.add("/bin/bash");
-        pargs.add(client.getEventsDir() + File.separator + "scripts" + File.separator + EXECUTE_SCRIPT);
+        pargs.add(client.getEventsHomeDir() + File.separator + AsterixEventServiceUtil.EVENT_DIR + File.separator
+                + EXECUTE_SCRIPT);
         StringBuffer envBuffer = new StringBuffer(IP_LOCATION + "=" + node.getClusterIp() + " ");
         boolean isMasterNode = node.getId().equals(cluster.getMasterNode().getId());
 
diff --git a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventTask.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventTask.java
index a764f9a..f98230b 100644
--- a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventTask.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventTask.java
@@ -20,6 +20,7 @@
 import java.util.List;
 import java.util.Timer;
 import java.util.TimerTask;
+
 import org.apache.log4j.Logger;
 
 import edu.uci.ics.asterix.event.driver.EventDriver;
@@ -30,134 +31,124 @@
 
 public class EventTask extends TimerTask {
 
-	public static enum State {
-		INITIALIZED, IN_PROGRESS, COMPLETED, FAILED
-	}
+    public static enum State {
+        INITIALIZED,
+        IN_PROGRESS,
+        COMPLETED,
+        FAILED
+    }
 
-	private static final Logger logger = Logger.getLogger(EventTask.class
-			.getName());
+    private static final Logger logger = Logger.getLogger(EventTask.class.getName());
 
-	private Pattern pattern;
-	private Event event;
-	private long interval = 0;
-	private long initialDelay = 0;
-	private int maxOccurs = Integer.MAX_VALUE;
-	private int occurrenceCount = 0;
-	private Timer timer;
-	private String taskScript;
-	private Node location;
-	private List<String> taskArgs;
-	private EventrixClient client;
-	private List<Node> candidateLocations;
-	private boolean dynamicLocation = false;
-	private boolean reuseLocation = false;
-	private State state;
+    private Pattern pattern;
+    private Event event;
+    private long interval = 0;
+    private long initialDelay = 0;
+    private int maxOccurs = Integer.MAX_VALUE;
+    private int occurrenceCount = 0;
+    private Timer timer;
+    private String taskScript;
+    private Node location;
+    private List<String> taskArgs;
+    private AsterixEventServiceClient client;
+    private List<Node> candidateLocations;
+    private boolean dynamicLocation = false;
+    private boolean reuseLocation = false;
+    private State state;
 
-	
+    public EventTask(Pattern pattern, AsterixEventServiceClient client) {
+        this.pattern = pattern;
+        this.client = client;
+        Period period = pattern.getPeriod();
+        if (period != null && period.getAbsvalue() != null) {
+            this.interval = EventUtil.parseTimeInterval(period.getAbsvalue(), period.getUnit());
+        }
+        if (pattern.getDelay() != null) {
+            this.initialDelay = EventUtil.parseTimeInterval(new ValueType(pattern.getDelay().getValue()), pattern
+                    .getDelay().getUnit());
+        }
+        if (pattern.getMaxOccurs() != null) {
+            this.maxOccurs = pattern.getMaxOccurs();
+        }
+        this.timer = new Timer();
+        taskArgs = EventUtil.getEventArgs(pattern);
+        candidateLocations = EventUtil.getCandidateLocations(pattern, client.getCluster());
+        if (pattern.getEvent().getNodeid().getValue().getRandom() != null && period != null && maxOccurs > 1) {
+            dynamicLocation = true;
+            reuseLocation = pattern.getEvent().getNodeid().getValue().getRandom().getRange().isReuse();
+        } else {
+            location = EventUtil.getEventLocation(pattern, candidateLocations, client.getCluster());
+        }
+        String scriptsDir;
+        if (location.getId().equals(EventDriver.CLIENT_NODE_ID)) {
+            scriptsDir = client.getEventsDir();
+        } else {
+            scriptsDir = client.getCluster().getWorkingDir().getDir() + File.separator + "events";
+        }
+        event = EventUtil.getEvent(pattern, client.getEvents());
+        taskScript = scriptsDir + File.separator + event.getScript();
+        state = State.INITIALIZED;
+    }
 
-	public EventTask(Pattern pattern, EventrixClient client) {
-		this.pattern = pattern;
-		this.client = client;
-		Period period = pattern.getPeriod();
-		if (period != null && period.getAbsvalue() != null) {
-			this.interval = EventUtil.parseTimeInterval(period.getAbsvalue(),
-					period.getUnit());
-		}
-		if (pattern.getDelay() != null) {
-			this.initialDelay = EventUtil.parseTimeInterval(new ValueType(
-					pattern.getDelay().getValue()), pattern.getDelay()
-					.getUnit());
-		}
-		if (pattern.getMaxOccurs() != null) {
-			this.maxOccurs = pattern.getMaxOccurs();
-		}
-		this.timer = new Timer();
-		taskArgs = EventUtil.getEventArgs(pattern);
-		candidateLocations = EventUtil.getCandidateLocations(pattern,
-				client.getCluster());
-		if (pattern.getEvent().getNodeid().getValue().getRandom() != null
-				&& period != null && maxOccurs > 1) {
-			dynamicLocation = true;
-			reuseLocation = pattern.getEvent().getNodeid().getValue()
-					.getRandom().getRange().isReuse();
-		} else {
-			location = EventUtil.getEventLocation(pattern, candidateLocations,
-					client.getCluster());
-		}
-		String scriptsDir;
-		if (location.getId().equals(EventDriver.CLIENT_NODE_ID)) {
-			scriptsDir = client.getEventsDir() + File.separator + "events";
-		} else {
-			scriptsDir = client.getCluster().getWorkingDir().getDir()
-					+ File.separator + "eventrix" + File.separator + "events";
-		}
-		event = EventUtil.getEvent(pattern, client.getEvents());
-		taskScript = scriptsDir + File.separator + event.getScript();
-		state = State.INITIALIZED;
-	}
+    public void start() {
+        if (interval > 0) {
+            timer.schedule(this, initialDelay, interval);
+        } else {
+            timer.schedule(this, initialDelay);
+        }
+    }
 
-	public void start() {
-		if (interval > 0) {
-			timer.schedule(this, initialDelay, interval);
-		} else {
-			timer.schedule(this, initialDelay);
-		}
-	}
+    @Override
+    public void run() {
+        if (candidateLocations.size() == 0) {
+            timer.cancel();
+            client.notifyCompletion(new EventTaskReport(this));
+        } else {
+            if (dynamicLocation) {
+                location = EventUtil.getEventLocation(pattern, candidateLocations, client.getCluster());
+                if (!reuseLocation) {
+                    candidateLocations.remove(location);
+                }
+            }
 
-	@Override
-	public void run() {
-		if (candidateLocations.size() == 0) {
-			timer.cancel();
-			client.notifyCompletion(new EventTaskReport(this));
-		} else {
-			if (dynamicLocation) {
-				location = EventUtil.getEventLocation(pattern,
-						candidateLocations, client.getCluster());
-				if (!reuseLocation) {
-					candidateLocations.remove(location);
-				}
-			}
+            logger.debug(EventUtil.dateFormat.format(new Date()) + " " + "EVENT "
+                    + pattern.getEvent().getType().toUpperCase() + " at " + location.getId().toUpperCase());
+            try {
+                if (!client.isDryRun()) {
+                    new EventExecutor().executeEvent(location, taskScript, taskArgs, event.isDaemon(),
+                            client.getCluster(), pattern, client.getErrorHandler(), client);
+                }
+                occurrenceCount++;
+                if (occurrenceCount >= maxOccurs) {
+                    timer.cancel();
+                    client.notifyCompletion(new EventTaskReport(this));
+                }
+            } catch (IOException ioe) {
+                timer.cancel();
+                client.notifyCompletion(new EventTaskReport(this, false, ioe));
+            }
+        }
 
-			logger.debug(EventUtil.dateFormat.format(new Date()) + " "
-					+ "EVENT " + pattern.getEvent().getType().toUpperCase()
-					+ " at " + location.getId().toUpperCase());
-			try {
-				if (!client.isDryRun()) {
-					new EventExecutor().executeEvent(location, taskScript,
-							taskArgs, event.isDaemon(), client.getCluster(),
-							pattern, client.getErrorHandler(), client);
-				}
-				occurrenceCount++;
-				if (occurrenceCount >= maxOccurs) {
-					timer.cancel();
-					client.notifyCompletion(new EventTaskReport(this));
-				}
-			} catch (IOException ioe) {
-				timer.cancel();
-				client.notifyCompletion(new EventTaskReport(this, false, ioe));
-			}
-		}
+    }
 
-	}
+    public Node getLocation() {
+        return location;
+    }
 
-	public Node getLocation() {
-		return location;
-	}
+    public long getInterval() {
+        return interval;
+    }
 
-	public long getInterval() {
-		return interval;
-	}
+    public long getInitialDelay() {
+        return initialDelay;
+    }
 
-	public long getInitialDelay() {
-		return initialDelay;
-	}
+    public Pattern getPattern() {
+        return pattern;
+    }
 
-	public Pattern getPattern() {
-		return pattern;
-	}
-
-	public State getState() {
-		return state;
-	}
+    public State getState() {
+        return state;
+    }
 
 }
diff --git a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventUtil.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventUtil.java
index 887e272..0e2fe92 100644
--- a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventUtil.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventUtil.java
@@ -16,7 +16,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.math.BigInteger;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
diff --git a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/IOutputHandler.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/IOutputHandler.java
index f6cf905..5dd370c 100644
--- a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/IOutputHandler.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/IOutputHandler.java
@@ -16,6 +16,7 @@
 
 import edu.uci.ics.asterix.event.schema.pattern.Event;
 
+
 public interface IOutputHandler {
 
     public OutputAnalysis reportEventOutput(Event event, String output);
diff --git a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/ValueType.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/ValueType.java
index b619002..2524d9a 100644
--- a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/ValueType.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/ValueType.java
@@ -16,6 +16,7 @@
 
 import edu.uci.ics.asterix.event.schema.pattern.Value;
 
+
 public class ValueType {
 
 	public static enum Type {
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/AsterixInstance.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/model/AsterixInstance.java
similarity index 98%
rename from asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/AsterixInstance.java
rename to asterix-events/src/main/java/edu/uci/ics/asterix/event/model/AsterixInstance.java
index c9a4743..b5e2460 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/AsterixInstance.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/model/AsterixInstance.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.installer.model;
+package edu.uci.ics.asterix.event.model;
 
 import java.io.Serializable;
 import java.util.ArrayList;
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/AsterixRuntimeState.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/model/AsterixRuntimeState.java
similarity index 96%
rename from asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/AsterixRuntimeState.java
rename to asterix-events/src/main/java/edu/uci/ics/asterix/event/model/AsterixRuntimeState.java
index e1a5167..efada48 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/AsterixRuntimeState.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/model/AsterixRuntimeState.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.installer.model;
+package edu.uci.ics.asterix.event.model;
 
 import java.io.Serializable;
 import java.util.List;
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/BackupInfo.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/model/BackupInfo.java
similarity index 97%
rename from asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/BackupInfo.java
rename to asterix-events/src/main/java/edu/uci/ics/asterix/event/model/BackupInfo.java
index c88fddb..e1bcd8a 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/BackupInfo.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/model/BackupInfo.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.installer.model;
+package edu.uci.ics.asterix.event.model;
 
 import java.io.Serializable;
 import java.util.Date;
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/EventList.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/model/EventList.java
similarity index 95%
rename from asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/EventList.java
rename to asterix-events/src/main/java/edu/uci/ics/asterix/event/model/EventList.java
index 91592e7..fda814c 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/EventList.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/model/EventList.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.installer.model;
+package edu.uci.ics.asterix.event.model;
 
 public class EventList {
 
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/ProcessInfo.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/model/ProcessInfo.java
similarity index 96%
rename from asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/ProcessInfo.java
rename to asterix-events/src/main/java/edu/uci/ics/asterix/event/model/ProcessInfo.java
index 56dfc8a..4fde136 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/ProcessInfo.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/model/ProcessInfo.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.installer.model;
+package edu.uci.ics.asterix.event.model;
 
 import java.io.Serializable;
 
diff --git a/asterix-events/src/main/java/edu/uci/ics/asterix/event/service/AsterixEventService.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/service/AsterixEventService.java
new file mode 100644
index 0000000..6744746
--- /dev/null
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/service/AsterixEventService.java
@@ -0,0 +1,78 @@
+package edu.uci.ics.asterix.event.service;
+
+import java.io.File;
+import java.io.FileFilter;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import edu.uci.ics.asterix.event.error.OutputHandler;
+import edu.uci.ics.asterix.event.management.AsterixEventServiceClient;
+import edu.uci.ics.asterix.event.schema.cluster.Cluster;
+import edu.uci.ics.asterix.installer.schema.conf.Configuration;
+
+public class AsterixEventService {
+
+    private static final Logger LOGGER = Logger.getLogger(AsterixEventService.class.getName());
+    private static Configuration configuration;
+    private static String asterixDir;
+    private static String asterixZip;
+    private static String eventHome;
+
+    public static void initialize(Configuration configuration, String asterixDir, String eventHome) throws Exception {
+        AsterixEventService.configuration = configuration;
+        AsterixEventService.asterixDir = asterixDir;
+        AsterixEventService.asterixZip = initBinary("asterix-server");
+        AsterixEventService.eventHome = eventHome;
+    }
+
+    public static AsterixEventServiceClient getAsterixEventServiceClient(Cluster cluster, boolean transferArtifacts,
+            boolean dryRun) throws Exception {
+        AsterixEventServiceClient client = new AsterixEventServiceClient(configuration, eventHome, cluster,
+                transferArtifacts, dryRun, OutputHandler.INSTANCE);
+        return client;
+    }
+
+    public static AsterixEventServiceClient getAsterixEventServiceClient(Cluster cluster) throws Exception {
+        AsterixEventServiceClient client = new AsterixEventServiceClient(configuration, eventHome, cluster, false,
+                false, OutputHandler.INSTANCE);
+        return client;
+    }
+
+    private static String initBinary(final String fileNamePattern) {
+        File file = new File(asterixDir);
+        File[] zipFiles = file.listFiles(new FileFilter() {
+            public boolean accept(File arg0) {
+                return arg0.getAbsolutePath().contains(fileNamePattern) && arg0.isFile();
+            }
+        });
+        if (zipFiles.length == 0) {
+            String msg = " Binary not found at " + asterixDir;
+            LOGGER.log(Level.FATAL, msg);
+            throw new IllegalStateException(msg);
+        }
+        if (zipFiles.length > 1) {
+            String msg = " Multiple binaries found at " + asterixDir;
+            LOGGER.log(Level.FATAL, msg);
+            throw new IllegalStateException(msg);
+        }
+
+        return zipFiles[0].getAbsolutePath();
+    }
+
+    public static Configuration getConfiguration() {
+        return configuration;
+    }
+
+    public static String getAsterixZip() {
+        return asterixZip;
+    }
+
+    public static String getAsterixDir() {
+        return asterixDir;
+    }
+
+    public static String getEventHome() {
+        return eventHome;
+    }
+}
diff --git a/asterix-events/src/main/java/edu/uci/ics/asterix/event/service/AsterixEventServiceUtil.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/service/AsterixEventServiceUtil.java
new file mode 100644
index 0000000..1bca9e9
--- /dev/null
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/service/AsterixEventServiceUtil.java
@@ -0,0 +1,525 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.event.service;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.JarOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+
+import org.apache.commons.io.IOUtils;
+
+import edu.uci.ics.asterix.common.configuration.AsterixConfiguration;
+import edu.uci.ics.asterix.common.configuration.Coredump;
+import edu.uci.ics.asterix.common.configuration.Store;
+import edu.uci.ics.asterix.common.configuration.TransactionLogDir;
+import edu.uci.ics.asterix.event.driver.EventDriver;
+import edu.uci.ics.asterix.event.error.EventException;
+import edu.uci.ics.asterix.event.management.EventUtil;
+import edu.uci.ics.asterix.event.model.AsterixInstance;
+import edu.uci.ics.asterix.event.model.AsterixInstance.State;
+import edu.uci.ics.asterix.event.schema.cluster.Cluster;
+import edu.uci.ics.asterix.event.schema.cluster.Env;
+import edu.uci.ics.asterix.event.schema.cluster.Node;
+import edu.uci.ics.asterix.event.schema.cluster.Property;
+
+public class AsterixEventServiceUtil {
+
+    public static final String TXN_LOG_DIR = "txnLogs";
+    public static final String TXN_LOG_DIR_KEY_SUFFIX = "txnLogDir";
+    public static final String ASTERIX_CONFIGURATION_FILE = "asterix-configuration.xml";
+    public static final String TXN_LOG_CONFIGURATION_FILE = "log.properties";
+    public static final String CLUSTER_CONFIGURATION_FILE = "cluster.xml";
+    public static final String ASTERIX_DIR = "asterix";
+    public static final String EVENT_DIR = "events";
+    public static final String DEFAULT_ASTERIX_CONFIGURATION_PATH = "conf" + File.separator + File.separator
+            + "asterix-configuration.xml";
+    public static final int CLUSTER_NET_PORT_DEFAULT = 1098;
+    public static final int CLIENT_NET_PORT_DEFAULT = 1099;
+    public static final int HTTP_PORT_DEFAULT = 8888;
+    public static final int WEB_INTERFACE_PORT_DEFAULT = 19001;
+
+    public static final String MANAGIX_INTERNAL_DIR = ".installer";
+    public static final String MANAGIX_CONF_XML = "conf" + File.separator + "managix-conf.xml";
+
+    public static AsterixInstance createAsterixInstance(String asterixInstanceName, Cluster cluster,
+            AsterixConfiguration asterixConfiguration) throws FileNotFoundException, IOException {
+        Node metadataNode = getMetadataNode(cluster);
+        String asterixZipName = AsterixEventService.getAsterixZip().substring(
+                AsterixEventService.getAsterixZip().lastIndexOf(File.separator) + 1);
+        String asterixVersion = asterixZipName.substring("asterix-server-".length(),
+                asterixZipName.indexOf("-binary-assembly"));
+        AsterixInstance instance = new AsterixInstance(asterixInstanceName, cluster, asterixConfiguration,
+                metadataNode.getId(), asterixVersion);
+        return instance;
+    }
+
+    public static void createAsterixZip(AsterixInstance asterixInstance) throws IOException, InterruptedException,
+            JAXBException, EventException {
+
+        String modifiedZipPath = injectAsterixPropertyFile(AsterixEventService.getAsterixZip(), asterixInstance);
+        modifiedZipPath = injectAsterixLogPropertyFile(modifiedZipPath, asterixInstance);
+        modifiedZipPath = injectAsterixClusterConfigurationFile(modifiedZipPath, asterixInstance);
+    }
+
+    public static void createClusterProperties(Cluster cluster, AsterixConfiguration asterixConfiguration) {
+        List<Property> clusterProperties = null;
+        if (cluster.getEnv() != null && cluster.getEnv().getProperty() != null) {
+            clusterProperties = cluster.getEnv().getProperty();
+            clusterProperties.clear();
+        } else {
+            clusterProperties = new ArrayList<Property>();
+        }
+        for (edu.uci.ics.asterix.common.configuration.Property property : asterixConfiguration.getProperty()) {
+            if (property.getName().equalsIgnoreCase(EventUtil.CC_JAVA_OPTS)) {
+                clusterProperties.add(new Property(EventUtil.CC_JAVA_OPTS, property.getValue()));
+            } else if (property.getName().equalsIgnoreCase(EventUtil.NC_JAVA_OPTS)) {
+                clusterProperties.add(new Property(EventUtil.NC_JAVA_OPTS, property.getValue()));
+            }
+        }
+        clusterProperties.add(new Property("ASTERIX_HOME", cluster.getWorkingDir().getDir() + File.separator
+                + "asterix"));
+        clusterProperties.add(new Property("LOG_DIR", cluster.getLogDir()));
+        clusterProperties.add(new Property("JAVA_HOME", cluster.getJavaHome()));
+        clusterProperties.add(new Property("WORKING_DIR", cluster.getWorkingDir().getDir()));
+        clusterProperties.add(new Property("CLIENT_NET_IP", cluster.getMasterNode().getClientIp()));
+        clusterProperties.add(new Property("CLUSTER_NET_IP", cluster.getMasterNode().getClusterIp()));
+
+        int clusterNetPort = cluster.getMasterNode().getClusterPort() != null ? cluster.getMasterNode()
+                .getClusterPort().intValue() : CLUSTER_NET_PORT_DEFAULT;
+        int clientNetPort = cluster.getMasterNode().getClientPort() != null ? cluster.getMasterNode().getClientPort()
+                .intValue() : CLIENT_NET_PORT_DEFAULT;
+        int httpPort = cluster.getMasterNode().getHttpPort() != null ? cluster.getMasterNode().getHttpPort().intValue()
+                : HTTP_PORT_DEFAULT;
+
+        clusterProperties.add(new Property("CLIENT_NET_PORT", "" + clientNetPort));
+        clusterProperties.add(new Property("CLUSTER_NET_PORT", "" + clusterNetPort));
+        clusterProperties.add(new Property("HTTP_PORT", "" + httpPort));
+
+        cluster.setEnv(new Env(clusterProperties));
+    }
+
+    public static void poulateClusterEnvironmentProperties(Cluster cluster) {
+
+    }
+
+    private static String injectAsterixPropertyFile(String origZipFile, AsterixInstance asterixInstance)
+            throws IOException, JAXBException {
+        writeAsterixConfigurationFile(asterixInstance);
+        String asterixInstanceDir = AsterixEventService.getAsterixDir() + File.separator + asterixInstance.getName();
+        unzip(origZipFile, asterixInstanceDir);
+        File sourceJar = new File(asterixInstanceDir + File.separator + "lib" + File.separator + "asterix-app-"
+                + asterixInstance.getAsterixVersion() + ".jar");
+        File replacementFile = new File(asterixInstanceDir + File.separator + ASTERIX_CONFIGURATION_FILE);
+        replaceInJar(sourceJar, ASTERIX_CONFIGURATION_FILE, replacementFile);
+        new File(asterixInstanceDir + File.separator + ASTERIX_CONFIGURATION_FILE).delete();
+        String asterixZipName = AsterixEventService.getAsterixZip().substring(
+                AsterixEventService.getAsterixZip().lastIndexOf(File.separator) + 1);
+        zipDir(new File(asterixInstanceDir), new File(asterixInstanceDir + File.separator + asterixZipName));
+        return asterixInstanceDir + File.separator + asterixZipName;
+    }
+
+    private static String injectAsterixLogPropertyFile(String origZipFile, AsterixInstance asterixInstance)
+            throws IOException, EventException {
+        String asterixInstanceDir = AsterixEventService.getAsterixDir() + File.separator + asterixInstance.getName();
+        unzip(origZipFile, asterixInstanceDir);
+        File sourceJar1 = new File(asterixInstanceDir + File.separator + "lib" + File.separator + "asterix-app-"
+                + asterixInstance.getAsterixVersion() + ".jar");
+        Properties txnLogProperties = new Properties();
+        URLClassLoader urlClassLoader = new URLClassLoader(new URL[] { sourceJar1.toURI().toURL() });
+        InputStream in = urlClassLoader.getResourceAsStream(TXN_LOG_CONFIGURATION_FILE);
+        if (in != null) {
+            txnLogProperties.load(in);
+        }
+
+        writeAsterixLogConfigurationFile(asterixInstance, txnLogProperties);
+
+        File sourceJar2 = new File(asterixInstanceDir + File.separator + "lib" + File.separator + "asterix-app-"
+                + asterixInstance.getAsterixVersion() + ".jar");
+        File replacementFile = new File(asterixInstanceDir + File.separator + "log.properties");
+        replaceInJar(sourceJar2, TXN_LOG_CONFIGURATION_FILE, replacementFile);
+
+        new File(asterixInstanceDir + File.separator + "log.properties").delete();
+        String asterixZipName = AsterixEventService.getAsterixZip().substring(
+                AsterixEventService.getAsterixZip().lastIndexOf(File.separator) + 1);
+        zipDir(new File(asterixInstanceDir), new File(asterixInstanceDir + File.separator + asterixZipName));
+        return asterixInstanceDir + File.separator + asterixZipName;
+    }
+
+    private static String injectAsterixClusterConfigurationFile(String origZipFile, AsterixInstance asterixInstance)
+            throws IOException, EventException, JAXBException {
+        String asterixInstanceDir = AsterixEventService.getAsterixDir() + File.separator + asterixInstance.getName();
+        unzip(origZipFile, asterixInstanceDir);
+        File sourceJar = new File(asterixInstanceDir + File.separator + "lib" + File.separator + "asterix-app-"
+                + asterixInstance.getAsterixVersion() + ".jar");
+        writeAsterixClusterConfigurationFile(asterixInstance);
+
+        File replacementFile = new File(asterixInstanceDir + File.separator + "cluster.xml");
+        replaceInJar(sourceJar, CLUSTER_CONFIGURATION_FILE, replacementFile);
+
+        new File(asterixInstanceDir + File.separator + CLUSTER_CONFIGURATION_FILE).delete();
+        String asterixZipName = AsterixEventService.getAsterixZip().substring(
+                AsterixEventService.getAsterixZip().lastIndexOf(File.separator) + 1);
+        zipDir(new File(asterixInstanceDir), new File(asterixInstanceDir + File.separator + asterixZipName));
+        return asterixInstanceDir + File.separator + asterixZipName;
+    }
+
+    private static void writeAsterixClusterConfigurationFile(AsterixInstance asterixInstance) throws IOException,
+            EventException, JAXBException {
+        String asterixInstanceName = asterixInstance.getName();
+        Cluster cluster = asterixInstance.getCluster();
+
+        JAXBContext ctx = JAXBContext.newInstance(Cluster.class);
+        Marshaller marshaller = ctx.createMarshaller();
+        marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+        marshaller.marshal(cluster, new FileOutputStream(AsterixEventService.getAsterixDir() + File.separator
+                + asterixInstanceName + File.separator + "cluster.xml"));
+    }
+
+    public static void addLibraryToAsterixZip(AsterixInstance asterixInstance, String dataverseName,
+            String libraryName, String libraryPath) throws IOException {
+        File instanceDir = new File(AsterixEventService.getAsterixDir() + File.separator + asterixInstance.getName());
+        if (!instanceDir.exists()) {
+            instanceDir.mkdirs();
+        }
+        String asterixZipName = AsterixEventService.getAsterixZip().substring(
+                AsterixEventService.getAsterixZip().lastIndexOf(File.separator) + 1);
+
+        String sourceZip = instanceDir.getAbsolutePath() + File.separator + asterixZipName;
+        unzip(sourceZip, instanceDir.getAbsolutePath());
+        File libraryPathInZip = new File(instanceDir.getAbsolutePath() + File.separator + "external" + File.separator
+                + "library" + dataverseName + File.separator + "to-add" + File.separator + libraryName);
+        libraryPathInZip.mkdirs();
+        Runtime.getRuntime().exec("cp" + " " + libraryPath + " " + libraryPathInZip.getAbsolutePath());
+        Runtime.getRuntime().exec("rm " + sourceZip);
+        String destZip = AsterixEventService.getAsterixDir() + File.separator + asterixInstance.getName()
+                + File.separator + asterixZipName;
+        zipDir(instanceDir, new File(destZip));
+        Runtime.getRuntime().exec("mv" + " " + destZip + " " + sourceZip);
+    }
+
+    private static Node getMetadataNode(Cluster cluster) {
+        Random random = new Random();
+        int nNodes = cluster.getNode().size();
+        return cluster.getNode().get(random.nextInt(nNodes));
+    }
+
+    public static String getNodeDirectories(String asterixInstanceName, Node node, Cluster cluster) {
+        String storeDataSubDir = asterixInstanceName + File.separator + "data" + File.separator;
+        String[] storeDirs = null;
+        StringBuffer nodeDataStore = new StringBuffer();
+        String storeDirValue = node.getStore();
+        if (storeDirValue == null) {
+            storeDirValue = cluster.getStore();
+            if (storeDirValue == null) {
+                throw new IllegalStateException(" Store not defined for node " + node.getId());
+            }
+            storeDataSubDir = node.getId() + File.separator + storeDataSubDir;
+        }
+
+        storeDirs = storeDirValue.split(",");
+        for (String ns : storeDirs) {
+            nodeDataStore.append(ns + File.separator + storeDataSubDir.trim());
+            nodeDataStore.append(",");
+        }
+        nodeDataStore.deleteCharAt(nodeDataStore.length() - 1);
+        return nodeDataStore.toString();
+    }
+
+    private static void writeAsterixConfigurationFile(AsterixInstance asterixInstance) throws IOException,
+            JAXBException {
+        String asterixInstanceName = asterixInstance.getName();
+        Cluster cluster = asterixInstance.getCluster();
+        String metadataNodeId = asterixInstance.getMetadataNodeId();
+
+        AsterixConfiguration configuration = asterixInstance.getAsterixConfiguration();
+        configuration.setInstanceName(asterixInstanceName);
+        configuration.setMetadataNode(asterixInstanceName + "_" + metadataNodeId);
+        String storeDir = null;
+        List<Store> stores = new ArrayList<Store>();
+        for (Node node : cluster.getNode()) {
+            storeDir = node.getStore() == null ? cluster.getStore() : node.getStore();
+            stores.add(new Store(asterixInstanceName + "_" + node.getId(), storeDir));
+        }
+        configuration.setStore(stores);
+
+        List<Coredump> coredump = new ArrayList<Coredump>();
+        String coredumpDir = null;
+        List<TransactionLogDir> txnLogDirs = new ArrayList<TransactionLogDir>();
+        String txnLogDir = null;
+        for (Node node : cluster.getNode()) {
+            coredumpDir = node.getLogDir() == null ? cluster.getLogDir() : node.getLogDir();
+            coredump.add(new Coredump(asterixInstanceName + "_" + node.getId(), coredumpDir + File.separator
+                    + asterixInstanceName + "_" + node.getId()));
+
+            txnLogDir = node.getTxnLogDir() == null ? cluster.getTxnLogDir() : node.getTxnLogDir();
+            txnLogDirs.add(new TransactionLogDir(asterixInstanceName + "_" + node.getId(), txnLogDir));
+        }
+        configuration.setCoredump(coredump);
+        configuration.setTransactionLogDir(txnLogDirs);
+
+        File asterixConfDir = new File(AsterixEventService.getAsterixDir() + File.separator + asterixInstanceName);
+        asterixConfDir.mkdirs();
+
+        JAXBContext ctx = JAXBContext.newInstance(AsterixConfiguration.class);
+        Marshaller marshaller = ctx.createMarshaller();
+        marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+        marshaller.marshal(configuration, new FileOutputStream(asterixConfDir + File.separator
+                + ASTERIX_CONFIGURATION_FILE));
+    }
+
+    private static void writeAsterixLogConfigurationFile(AsterixInstance asterixInstance, Properties logProperties)
+            throws IOException, EventException {
+        String asterixInstanceName = asterixInstance.getName();
+        Cluster cluster = asterixInstance.getCluster();
+        StringBuffer conf = new StringBuffer();
+        for (Map.Entry<Object, Object> p : logProperties.entrySet()) {
+            conf.append(p.getKey() + "=" + p.getValue() + "\n");
+        }
+
+        for (Node node : cluster.getNode()) {
+            String txnLogDir = node.getTxnLogDir() == null ? cluster.getTxnLogDir() : node.getTxnLogDir();
+            if (txnLogDir == null) {
+                throw new EventException("Transaction log directory (txn_log_dir) not configured for node: "
+                        + node.getId());
+            }
+            conf.append(asterixInstanceName + "_" + node.getId() + "." + TXN_LOG_DIR_KEY_SUFFIX + "=" + txnLogDir
+                    + "\n");
+        }
+        List<edu.uci.ics.asterix.common.configuration.Property> properties = asterixInstance.getAsterixConfiguration()
+                .getProperty();
+        for (edu.uci.ics.asterix.common.configuration.Property p : properties) {
+            if (p.getName().trim().toLowerCase().contains("log")) {
+                conf.append(p.getValue() + "=" + p.getValue());
+            }
+        }
+        dumpToFile(AsterixEventService.getAsterixDir() + File.separator + asterixInstanceName + File.separator
+                + "log.properties", conf.toString());
+
+    }
+
+    public static void unzip(String sourceFile, String destDir) throws IOException {
+        BufferedOutputStream dest = null;
+        FileInputStream fis = new FileInputStream(sourceFile);
+        ZipInputStream zis = new ZipInputStream(new BufferedInputStream(fis));
+        ZipEntry entry = null;
+
+        int BUFFER_SIZE = 4096;
+        while ((entry = zis.getNextEntry()) != null) {
+            String dst = destDir + File.separator + entry.getName();
+            if (entry.isDirectory()) {
+                createDir(destDir, entry);
+                continue;
+            }
+            int count;
+            byte data[] = new byte[BUFFER_SIZE];
+
+            // write the file to the disk
+            FileOutputStream fos = new FileOutputStream(dst);
+            dest = new BufferedOutputStream(fos, BUFFER_SIZE);
+            while ((count = zis.read(data, 0, BUFFER_SIZE)) != -1) {
+                dest.write(data, 0, count);
+            }
+            // close the output streams
+            dest.flush();
+            dest.close();
+        }
+
+        zis.close();
+    }
+
+    public static void zipDir(File sourceDir, File destFile) throws IOException {
+        FileOutputStream fos = new FileOutputStream(destFile);
+        ZipOutputStream zos = new ZipOutputStream(fos);
+        zipDir(sourceDir, destFile, zos);
+        zos.close();
+    }
+
+    private static void zipDir(File sourceDir, final File destFile, ZipOutputStream zos) throws IOException {
+        File[] dirList = sourceDir.listFiles(new FileFilter() {
+            public boolean accept(File f) {
+                return !f.getName().endsWith(destFile.getName());
+            }
+        });
+        for (int i = 0; i < dirList.length; i++) {
+            File f = dirList[i];
+            if (f.isDirectory()) {
+                zipDir(f, destFile, zos);
+            } else {
+                int bytesIn = 0;
+                byte[] readBuffer = new byte[2156];
+                FileInputStream fis = new FileInputStream(f);
+                ZipEntry entry = new ZipEntry(sourceDir.getName() + File.separator + f.getName());
+                zos.putNextEntry(entry);
+                while ((bytesIn = fis.read(readBuffer)) != -1) {
+                    zos.write(readBuffer, 0, bytesIn);
+                }
+                fis.close();
+            }
+        }
+    }
+
+    private static void replaceInJar(File sourceJar, String origFile, File replacementFile) throws IOException {
+        File destJar = new File(sourceJar.getAbsolutePath() + ".modified");
+        InputStream jarIs = null;
+        FileInputStream fis = new FileInputStream(replacementFile);
+        JarFile sourceJarFile = new JarFile(sourceJar);
+        Enumeration<JarEntry> entries = sourceJarFile.entries();
+        JarOutputStream jos = new JarOutputStream(new FileOutputStream(destJar));
+        byte[] buffer = new byte[2048];
+        int read;
+        while (entries.hasMoreElements()) {
+            JarEntry entry = (JarEntry) entries.nextElement();
+            String name = entry.getName();
+            if (name.equals(origFile)) {
+                continue;
+            }
+            jarIs = sourceJarFile.getInputStream(entry);
+            jos.putNextEntry(entry);
+            while ((read = jarIs.read(buffer)) != -1) {
+                jos.write(buffer, 0, read);
+            }
+        }
+        JarEntry entry = new JarEntry(origFile);
+        jos.putNextEntry(entry);
+        while ((read = fis.read(buffer)) != -1) {
+            jos.write(buffer, 0, read);
+        }
+        fis.close();
+        jos.close();
+        jarIs.close();
+        sourceJar.delete();
+        destJar.renameTo(sourceJar);
+        sourceJar.setExecutable(true);
+    }
+
+    public static void dumpToFile(String dest, String content) throws IOException {
+        FileWriter writer = new FileWriter(dest);
+        writer.write(content);
+        writer.close();
+    }
+
+    private static void createDir(String destDirectory, ZipEntry entry) {
+        String name = entry.getName();
+        int index = name.lastIndexOf(File.separator);
+        String dirSequence = name.substring(0, index);
+        File newDirs = new File(destDirectory + File.separator + dirSequence);
+        newDirs.mkdirs();
+    }
+
+    public static AsterixInstance validateAsterixInstanceExists(String name, State... permissibleStates)
+            throws Exception {
+        AsterixInstance instance = ServiceProvider.INSTANCE.getLookupService().getAsterixInstance(name);
+        if (instance == null) {
+            throw new EventException("Asterix instance by name " + name + " does not exist.");
+        }
+        boolean valid = false;
+        for (State state : permissibleStates) {
+            if (state.equals(instance.getState())) {
+                valid = true;
+                break;
+            }
+        }
+        if (!valid) {
+            throw new EventException("Asterix instance by the name " + name + " is in " + instance.getState()
+                    + " state ");
+        }
+        return instance;
+    }
+
+    public static void validateAsterixInstanceNotExists(String name) throws Exception {
+        AsterixInstance instance = ServiceProvider.INSTANCE.getLookupService().getAsterixInstance(name);
+        if (instance != null) {
+            throw new EventException("Asterix instance by name " + name + " already exists.");
+        }
+    }
+
+    public static void evaluateConflictWithOtherInstances(AsterixInstance instance) throws Exception {
+        List<AsterixInstance> existingInstances = ServiceProvider.INSTANCE.getLookupService().getAsterixInstances();
+        List<String> usedIps = new ArrayList<String>();
+        String masterIp = instance.getCluster().getMasterNode().getClusterIp();
+        for (Node node : instance.getCluster().getNode()) {
+            usedIps.add(node.getClusterIp());
+        }
+        usedIps.add(instance.getCluster().getMasterNode().getClusterIp());
+        boolean conflictFound = false;
+        AsterixInstance conflictingInstance = null;
+        for (AsterixInstance existing : existingInstances) {
+            conflictFound = existing.getCluster().getMasterNode().getClusterIp().equals(masterIp);
+            if (conflictFound) {
+                conflictingInstance = existing;
+                break;
+            }
+            for (Node n : existing.getCluster().getNode()) {
+                if (usedIps.contains(n.getClusterIp())) {
+                    conflictFound = true;
+                    conflictingInstance = existing;
+                    break;
+                }
+            }
+        }
+        if (conflictFound) {
+            throw new Exception("Cluster definition conflicts with an existing instance of Asterix: "
+                    + conflictingInstance.getName());
+        }
+    }
+
+    public static void deleteDirectory(String path) throws IOException {
+        Runtime.getRuntime().exec("rm -rf " + path);
+    }
+
+    public static String executeLocalScript(String path, List<String> args) throws Exception {
+        List<String> pargs = new ArrayList<String>();
+        pargs.add("/bin/bash");
+        pargs.add(path);
+        if (args != null) {
+            pargs.addAll(args);
+        }
+        ProcessBuilder pb = new ProcessBuilder(pargs);
+        pb.environment().putAll(EventDriver.getEnvironment());
+        pb.environment().put("IP_LOCATION", EventDriver.CLIENT_NODE.getClusterIp());
+        Process p = pb.start();
+        BufferedInputStream bis = new BufferedInputStream(p.getInputStream());
+        StringWriter writer = new StringWriter();
+        IOUtils.copy(bis, writer, "UTF-8");
+        return writer.toString();
+    }
+
+}
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/service/ILookupService.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/service/ILookupService.java
similarity index 92%
rename from asterix-installer/src/main/java/edu/uci/ics/asterix/installer/service/ILookupService.java
rename to asterix-events/src/main/java/edu/uci/ics/asterix/event/service/ILookupService.java
index 59fa198..cac504f 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/service/ILookupService.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/service/ILookupService.java
@@ -12,11 +12,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.installer.service;
+package edu.uci.ics.asterix.event.service;
 
 import java.util.List;
 
-import edu.uci.ics.asterix.installer.model.AsterixInstance;
+import edu.uci.ics.asterix.event.model.AsterixInstance;
 import edu.uci.ics.asterix.installer.schema.conf.Configuration;
 
 public interface ILookupService {
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/service/ServiceProvider.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/service/ServiceProvider.java
similarity index 94%
rename from asterix-installer/src/main/java/edu/uci/ics/asterix/installer/service/ServiceProvider.java
rename to asterix-events/src/main/java/edu/uci/ics/asterix/event/service/ServiceProvider.java
index 98d72f4..56e5337 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/service/ServiceProvider.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/service/ServiceProvider.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.installer.service;
+package edu.uci.ics.asterix.event.service;
 
 public class ServiceProvider {
 
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/service/ZooKeeperService.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/service/ZooKeeperService.java
similarity index 91%
rename from asterix-installer/src/main/java/edu/uci/ics/asterix/installer/service/ZooKeeperService.java
rename to asterix-events/src/main/java/edu/uci/ics/asterix/event/service/ZooKeeperService.java
index d3304d2..7063c76 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/service/ZooKeeperService.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/service/ZooKeeperService.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.installer.service;
+package edu.uci.ics.asterix.event.service;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -34,10 +34,8 @@
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.data.Stat;
 
-import edu.uci.ics.asterix.installer.driver.InstallerDriver;
-import edu.uci.ics.asterix.installer.driver.InstallerUtil;
-import edu.uci.ics.asterix.installer.error.InstallerException;
-import edu.uci.ics.asterix.installer.model.AsterixInstance;
+import edu.uci.ics.asterix.event.error.EventException;
+import edu.uci.ics.asterix.event.model.AsterixInstance;
 import edu.uci.ics.asterix.installer.schema.conf.Configuration;
 
 public class ZooKeeperService implements ILookupService {
@@ -47,8 +45,7 @@
     private static final int ZOOKEEPER_LEADER_CONN_PORT = 2222;
     private static final int ZOOKEEPER_LEADER_ELEC_PORT = 2223;
     private static final int ZOOKEEPER_SESSION_TIME_OUT = 40 * 1000; //milliseconds
-    private static final String ZOOKEEPER_HOME = InstallerDriver.getManagixHome() + File.separator
-            + InstallerDriver.MANAGIX_INTERNAL_DIR + File.separator + "zookeeper";
+    private static final String ZOOKEEPER_HOME = AsterixEventService.getEventHome() + File.separator + "zookeeper";
     private static final String ZOO_KEEPER_CONFIG = ZOOKEEPER_HOME + File.separator + "zk.cfg";
 
     private boolean isRunning = false;
@@ -109,10 +106,10 @@
         if (head == null) {
             StringBuilder msg = new StringBuilder(
                     "Unable to start Zookeeper Service. This could be because of the following reasons.\n");
-            msg.append("1) Managix is incorrectly configured. Please run " + InstallerDriver.getManagixHome()
-                    + "/bin/managix validate" + " to run a validation test and correct the errors reported.");
+            msg.append("1) Managix is incorrectly configured. Please run " + "managix validate"
+                    + " to run a validation test and correct the errors reported.");
             msg.append("\n2) If validation in (1) is successful, ensure that java_home parameter is set correctly in Managix configuration ("
-                    + InstallerDriver.getManagixHome() + File.separator + InstallerDriver.MANAGIX_CONF_XML + ")");
+                    + null + File.separator + AsterixEventServiceUtil.MANAGIX_CONF_XML + ")");
             throw new Exception(msg.toString());
         }
         msgQ.take();
@@ -172,7 +169,7 @@
 
     public void removeAsterixInstance(String name) throws Exception {
         if (!exists(name)) {
-            throw new InstallerException("Asterix instance by name " + name + " does not exists.");
+            throw new EventException("Asterix instance by name " + name + " does not exists.");
         }
         zk.delete(ASTERIX_INSTANCE_BASE_PATH + File.separator + name, DEFAULT_NODE_VERSION);
     }
@@ -244,7 +241,7 @@
             buffer.append("server" + "." + serverId + "=" + server + ":" + leaderConnPort + ":" + leaderElecPort + "\n");
             serverId++;
         }
-        InstallerUtil.dumpToFile(zooKeeperConfigPath, buffer.toString());
+        AsterixEventServiceUtil.dumpToFile(zooKeeperConfigPath, buffer.toString());
     }
 
 }
diff --git a/asterix-events/src/main/java/edu/uci/ics/asterix/event/util/AsterixConstants.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/util/AsterixConstants.java
new file mode 100644
index 0000000..ff600ce
--- /dev/null
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/util/AsterixConstants.java
@@ -0,0 +1,7 @@
+package edu.uci.ics.asterix.event.util;
+
+public class AsterixConstants {
+
+    public static String ASTERIX_ROOT_METADATA_DIR = "asterix_root_metadata";
+
+}
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/events/PatternCreator.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/util/PatternCreator.java
similarity index 82%
rename from asterix-installer/src/main/java/edu/uci/ics/asterix/installer/events/PatternCreator.java
rename to asterix-events/src/main/java/edu/uci/ics/asterix/event/util/PatternCreator.java
index fe61462..7054975 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/events/PatternCreator.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/util/PatternCreator.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.installer.events;
+package edu.uci.ics.asterix.event.util;
 
 import java.io.File;
 import java.util.ArrayList;
@@ -23,6 +23,10 @@
 import java.util.Set;
 
 import edu.uci.ics.asterix.event.driver.EventDriver;
+import edu.uci.ics.asterix.event.error.VerificationUtil;
+import edu.uci.ics.asterix.event.model.AsterixInstance;
+import edu.uci.ics.asterix.event.model.BackupInfo;
+import edu.uci.ics.asterix.event.model.BackupInfo.BackupType;
 import edu.uci.ics.asterix.event.schema.cluster.Cluster;
 import edu.uci.ics.asterix.event.schema.cluster.Node;
 import edu.uci.ics.asterix.event.schema.pattern.Delay;
@@ -31,20 +35,20 @@
 import edu.uci.ics.asterix.event.schema.pattern.Pattern;
 import edu.uci.ics.asterix.event.schema.pattern.Patterns;
 import edu.uci.ics.asterix.event.schema.pattern.Value;
-import edu.uci.ics.asterix.installer.command.BackupCommand;
-import edu.uci.ics.asterix.installer.command.StopCommand;
-import edu.uci.ics.asterix.installer.driver.InstallerDriver;
-import edu.uci.ics.asterix.installer.driver.InstallerUtil;
-import edu.uci.ics.asterix.installer.error.VerificationUtil;
-import edu.uci.ics.asterix.installer.model.AsterixInstance;
-import edu.uci.ics.asterix.installer.model.BackupInfo;
-import edu.uci.ics.asterix.installer.model.BackupInfo.BackupType;
+import edu.uci.ics.asterix.event.service.AsterixEventService;
+import edu.uci.ics.asterix.event.service.AsterixEventServiceUtil;
+import edu.uci.ics.asterix.event.service.ILookupService;
+import edu.uci.ics.asterix.event.service.ServiceProvider;
 import edu.uci.ics.asterix.installer.schema.conf.Backup;
-import edu.uci.ics.asterix.installer.service.ILookupService;
-import edu.uci.ics.asterix.installer.service.ServiceProvider;
 
 public class PatternCreator {
 
+    public static PatternCreator INSTANCE = new PatternCreator();
+
+    private PatternCreator() {
+
+    }
+
     private ILookupService lookupService = ServiceProvider.INSTANCE.getLookupService();
 
     private void addInitialDelay(Pattern p, int delay, String unit) {
@@ -94,9 +98,9 @@
         return patterns;
     }
 
-    public Patterns getStopCommandPattern(StopCommand stopCommand) throws Exception {
+    public Patterns getStopCommandPattern(String asterixInstanceName) throws Exception {
         List<Pattern> ps = new ArrayList<Pattern>();
-        AsterixInstance asterixInstance = lookupService.getAsterixInstance(stopCommand.getAsterixInstanceName());
+        AsterixInstance asterixInstance = lookupService.getAsterixInstance(asterixInstanceName);
         Cluster cluster = asterixInstance.getCluster();
 
         String ccLocation = cluster.getMasterNode().getId();
@@ -104,7 +108,6 @@
         addInitialDelay(createCC, 5, "sec");
         ps.add(createCC);
 
-        String asterixInstanceName = stopCommand.getAsterixInstanceName();
         int nodeControllerIndex = 1;
         for (Node node : cluster.getNode()) {
             Pattern createNC = createNCStopPattern(node.getId(), asterixInstanceName + "_" + nodeControllerIndex);
@@ -161,8 +164,9 @@
             iodevices = node.getIodevices() == null ? instance.getCluster().getIodevices() : node.getIodevices();
             store = node.getStore() == null ? cluster.getStore() : node.getStore();
             pargs = workingDir + " " + instance.getName() + " " + iodevices + " " + store + " "
-                    + BackupCommand.ASTERIX_ROOT_METADATA_DIR + " " + InstallerUtil.TXN_LOG_DIR + " " + backupId + " "
-                    + hdfsBackupDir + " " + "hdfs" + " " + node.getId() + " " + hdfsUrl + " " + hadoopVersion;
+                    + AsterixConstants.ASTERIX_ROOT_METADATA_DIR + " " + AsterixEventServiceUtil.TXN_LOG_DIR + " "
+                    + backupId + " " + hdfsBackupDir + " " + "hdfs" + " " + node.getId() + " " + hdfsUrl + " "
+                    + hadoopVersion;
             Event event = new Event("backup", nodeid, pargs);
             patternList.add(new Pattern(null, 1, null, event));
         }
@@ -185,7 +189,7 @@
             txnLogDir = node.getTxnLogDir() == null ? instance.getCluster().getTxnLogDir() : node.getTxnLogDir();
             store = node.getStore() == null ? cluster.getStore() : node.getStore();
             pargs = workingDir + " " + instance.getName() + " " + iodevices + " " + store + " "
-                    + BackupCommand.ASTERIX_ROOT_METADATA_DIR  + " " + txnLogDir + " " + backupId + " " + backupDir
+                    + AsterixConstants.ASTERIX_ROOT_METADATA_DIR + " " + txnLogDir + " " + backupId + " " + backupDir
                     + " " + "local" + " " + node.getId();
             Event event = new Event("backup", nodeid, pargs);
             patternList.add(new Pattern(null, 1, null, event));
@@ -210,8 +214,9 @@
             String iodevices = node.getIodevices() == null ? cluster.getIodevices() : node.getIodevices();
             nodeStore = node.getStore() == null ? clusterStore : node.getStore();
             pargs = workingDir + " " + instance.getName() + " " + iodevices + " " + nodeStore + " "
-                    + BackupCommand.ASTERIX_ROOT_METADATA_DIR + " " + InstallerUtil.TXN_LOG_DIR + " " + backupId + " "
-                    + " " + hdfsBackupDir + " " + "hdfs" + " " + node.getId() + " " + hdfsUrl + " " + hadoopVersion;
+                    + AsterixConstants.ASTERIX_ROOT_METADATA_DIR + " " + AsterixEventServiceUtil.TXN_LOG_DIR + " "
+                    + backupId + " " + " " + hdfsBackupDir + " " + "hdfs" + " " + node.getId() + " " + hdfsUrl + " "
+                    + hadoopVersion;
             Event event = new Event("restore", nodeid, pargs);
             patternList.add(new Pattern(null, 1, null, event));
         }
@@ -232,8 +237,8 @@
             String iodevices = node.getIodevices() == null ? cluster.getIodevices() : node.getIodevices();
             nodeStore = node.getStore() == null ? clusterStore : node.getStore();
             pargs = workingDir + " " + instance.getName() + " " + iodevices + " " + nodeStore + " "
-                    + BackupCommand.ASTERIX_ROOT_METADATA_DIR + " " + InstallerUtil.TXN_LOG_DIR + " " + backupId + " "
-                    + backupDir + " " + "local" + " " + node.getId();
+                    + AsterixConstants.ASTERIX_ROOT_METADATA_DIR + " " + AsterixEventServiceUtil.TXN_LOG_DIR + " "
+                    + backupId + " " + backupDir + " " + "local" + " " + node.getId();
             Event event = new Event("restore", nodeid, pargs);
             patternList.add(new Pattern(null, 1, null, event));
         }
@@ -243,9 +248,8 @@
     public Patterns createHadoopLibraryTransferPattern(Cluster cluster) throws Exception {
         List<Pattern> patternList = new ArrayList<Pattern>();
         String workingDir = cluster.getWorkingDir().getDir();
-        String hadoopVersion = InstallerDriver.getConfiguration().getBackup().getHdfs().getVersion();
-        File hadoopDir = new File(InstallerDriver.getManagixHome() + File.separator
-                + InstallerDriver.MANAGIX_INTERNAL_DIR + File.separator + "hadoop-" + hadoopVersion);
+        String hadoopVersion = AsterixEventService.getConfiguration().getBackup().getHdfs().getVersion();
+        File hadoopDir = new File(AsterixEventService.getEventHome() + File.separator + "hadoop-" + hadoopVersion);
         if (!hadoopDir.exists()) {
             throw new IllegalStateException("Hadoop version :" + hadoopVersion + " not supported");
         }
@@ -332,8 +336,8 @@
     private Patterns createRemoveHDFSBackupPattern(AsterixInstance instance, String hdfsBackupDir) throws Exception {
         List<Pattern> patternList = new ArrayList<Pattern>();
         Cluster cluster = instance.getCluster();
-        String hdfsUrl = InstallerDriver.getConfiguration().getBackup().getHdfs().getUrl();
-        String hadoopVersion = InstallerDriver.getConfiguration().getBackup().getHdfs().getVersion();
+        String hdfsUrl = AsterixEventService.getConfiguration().getBackup().getHdfs().getUrl();
+        String hadoopVersion = AsterixEventService.getConfiguration().getBackup().getHdfs().getVersion();
         String workingDir = cluster.getWorkingDir().getDir();
         Node launchingNode = cluster.getNode().get(0);
         Nodeid nodeid = new Nodeid(new Value(null, launchingNode.getId()));
@@ -395,7 +399,7 @@
         for (Node node : cluster.getNode()) {
             String iodevices = node.getIodevices() == null ? cluster.getIodevices() : node.getIodevices();
             String primaryIODevice = iodevices.split(",")[0].trim();
-            pargs = primaryIODevice + File.separator + BackupCommand.ASTERIX_ROOT_METADATA_DIR;
+            pargs = primaryIODevice + File.separator + AsterixConstants.ASTERIX_ROOT_METADATA_DIR;
             nodeid = new Nodeid(new Value(null, node.getId()));
             event = new Event("file_delete", nodeid, pargs);
             patternList.add(new Pattern(null, 1, null, event));
@@ -447,9 +451,9 @@
     private Pattern createCopyHyracksPattern(String instanceName, Cluster cluster, String destinationIp, String destDir) {
         Nodeid nodeid = new Nodeid(new Value(null, EventDriver.CLIENT_NODE.getId()));
         String username = cluster.getUsername() != null ? cluster.getUsername() : System.getProperty("user.name");
-        String asterixZipName = InstallerDriver.getAsterixZip().substring(
-                InstallerDriver.getAsterixZip().lastIndexOf(File.separator) + 1);
-        String fileToTransfer = new File(InstallerDriver.getAsterixDir() + File.separator + instanceName
+        String asterixZipName = AsterixEventService.getAsterixZip().substring(
+                AsterixEventService.getAsterixZip().lastIndexOf(File.separator) + 1);
+        String fileToTransfer = new File(AsterixEventService.getAsterixDir() + File.separator + instanceName
                 + File.separator + asterixZipName).getAbsolutePath();
         String pargs = username + " " + fileToTransfer + " " + destinationIp + " " + destDir + " " + "unpack";
         Event event = new Event("file_transfer", nodeid, pargs);
@@ -481,10 +485,48 @@
         return new Pattern(null, 1, null, event);
     }
 
+    public Patterns createPrepareNodePattern(String instanceName, Cluster cluster, Node nodeToBeAdded) {
+        List<Pattern> ps = new ArrayList<Pattern>();
+        boolean workingDirOnNFS = cluster.getWorkingDir().isNFS();
+        if (!workingDirOnNFS) {
+            String ccLocationIp = cluster.getMasterNode().getClusterIp();
+            String destDir = cluster.getWorkingDir().getDir() + File.separator + "asterix";
+            Pattern copyHyracks = createCopyHyracksPattern(instanceName, cluster, ccLocationIp, destDir);
+            ps.add(copyHyracks);
+
+            String workingDir = cluster.getWorkingDir().getDir();
+            String hadoopVersion = AsterixEventService.getConfiguration().getBackup().getHdfs().getVersion();
+            File hadoopDir = new File(AsterixEventService.getEventHome() + File.separator + "hadoop-" + hadoopVersion);
+            if (!hadoopDir.exists()) {
+                throw new IllegalStateException("Hadoop version :" + hadoopVersion + " not supported");
+            }
+
+            Nodeid nodeid = new Nodeid(new Value(null, EventDriver.CLIENT_NODE.getId()));
+            String username = cluster.getUsername() != null ? cluster.getUsername() : System.getProperty("user.name");
+            String pargs = username + " " + hadoopDir.getAbsolutePath() + " " + cluster.getMasterNode().getClusterIp()
+                    + " " + workingDir;
+            Event event = new Event("directory_transfer", nodeid, pargs);
+            Pattern p = new Pattern(null, 1, null, event);
+            addInitialDelay(p, 2, "sec");
+            ps.add(p);
+
+            nodeid = new Nodeid(new Value(null, nodeToBeAdded.getId()));
+            pargs = cluster.getUsername() + " " + hadoopDir.getAbsolutePath() + " " + nodeToBeAdded.getClusterIp()
+                    + " " + workingDir;
+            event = new Event("directory_transfer", nodeid, pargs);
+            p = new Pattern(null, 1, null, event);
+            addInitialDelay(p, 2, "sec");
+            ps.add(p);
+        }
+
+        Patterns patterns = new Patterns(ps);
+        return patterns;
+    }
+
     public Patterns getGenerateLogPattern(String asterixInstanceName, Cluster cluster, String outputDir) {
         List<Pattern> patternList = new ArrayList<Pattern>();
-        Map<String,String> nodeLogs = new HashMap<String,String>();
-        
+        Map<String, String> nodeLogs = new HashMap<String, String>();
+
         String username = cluster.getUsername() == null ? System.getProperty("user.name") : cluster.getUsername();
         String srcHost = cluster.getMasterNode().getClientIp();
         Nodeid nodeid = new Nodeid(new Value(null, EventDriver.CLIENT_NODE.getId()));
@@ -495,15 +537,15 @@
         Event event = new Event("directory_copy", nodeid, pargs);
         Pattern p = new Pattern(null, 1, null, event);
         patternList.add(p);
-        nodeLogs.put(cluster.getMasterNode().getClusterIp(),srcDir);
+        nodeLogs.put(cluster.getMasterNode().getClusterIp(), srcDir);
         for (Node node : cluster.getNode()) {
             srcHost = node.getClusterIp();
             srcDir = node.getLogDir() == null ? cluster.getLogDir() : node.getLogDir();
-            if(nodeLogs.get(node.getClusterIp()) != null && nodeLogs.get(node.getClusterIp()).equals(srcDir)){
+            if (nodeLogs.get(node.getClusterIp()) != null && nodeLogs.get(node.getClusterIp()).equals(srcDir)) {
                 continue;
             }
             destDir = outputDir + File.separator + node.getId();
-            pargs = username + " " + srcHost +  " " + srcDir + " "  + destDir;
+            pargs = username + " " + srcHost + " " + srcDir + " " + destDir;
             event = new Event("directory_copy", nodeid, pargs);
             p = new Pattern(null, 1, null, event);
             patternList.add(p);
diff --git a/asterix-events/src/main/resources/events/backup/backup.sh b/asterix-events/src/main/resources/events/backup/backup.sh
old mode 100755
new mode 100644
diff --git a/asterix-events/src/main/resources/events/cc_failure/cc_failure.sh b/asterix-events/src/main/resources/events/cc_failure/cc_failure.sh
old mode 100755
new mode 100644
diff --git a/asterix-events/src/main/resources/events/cc_start/cc_start.sh b/asterix-events/src/main/resources/events/cc_start/cc_start.sh
old mode 100755
new mode 100644
diff --git a/asterix-events/src/main/resources/events/execute.sh b/asterix-events/src/main/resources/events/execute.sh
new file mode 100644
index 0000000..30f6c2f
--- /dev/null
+++ b/asterix-events/src/main/resources/events/execute.sh
@@ -0,0 +1,27 @@
+USERNAME=$1
+if [ ! -d $MANAGIX_HOME/logs ];
+then
+   mkdir -p $MANAGIX_HOME/logs
+fi
+LOGDIR=$MANAGIX_HOME/logs
+if [ $DAEMON == "false" ]; then 
+  if [ -z $USERNAME ]
+  then
+    cmd_output=$(ssh $IP_LOCATION "$ENV $SCRIPT $ARGS" 2>&1 >/dev/null) 
+    echo "ssh $IP_LOCATION $ENV $SCRIPT $ARGS" >> $LOGDIR/execute.log
+    echo "$cmd_output"
+  else
+    echo "ssh -l $USERNAME $IP_LOCATION $ENV $SCRIPT $ARGS" >> $LOGDIR/execute.log
+    cmd_output=$(ssh -l $USERNAME $IP_LOCATION "$ENV $SCRIPT $ARGS" 2>&1 >/dev/null) 
+    echo "$cmd_output"
+  fi  
+else 
+  if [ -z $USERNAME ];
+  then
+     echo "ssh $IP_LOCATION $ENV $SCRIPT $ARGS &" >> $LOGDIR/execute.log
+     ssh $IP_LOCATION "$ENV $SCRIPT $ARGS" &
+  else
+     echo "ssh -l $USERNAME $IP_LOCATION $ENV $SCRIPT $ARGS &" >> $LOGDIR/execute.log
+     ssh -l $USERNAME $IP_LOCATION "$ENV $SCRIPT $ARGS" &
+  fi   
+fi
diff --git a/asterix-events/src/main/resources/events/file/delete.sh b/asterix-events/src/main/resources/events/file/delete.sh
old mode 100755
new mode 100644
diff --git a/asterix-events/src/main/resources/events/file/dir_transfer.sh b/asterix-events/src/main/resources/events/file/dir_transfer.sh
old mode 100755
new mode 100644
diff --git a/asterix-events/src/main/resources/events/file/transfer.sh b/asterix-events/src/main/resources/events/file/transfer.sh
old mode 100755
new mode 100644
diff --git a/asterix-events/src/main/resources/events/hdfs/delete.sh b/asterix-events/src/main/resources/events/hdfs/delete.sh
old mode 100755
new mode 100644
diff --git a/asterix-events/src/main/resources/events/node_failure/nc_failure.sh b/asterix-events/src/main/resources/events/node_failure/nc_failure.sh
old mode 100755
new mode 100644
diff --git a/asterix-events/src/main/resources/events/node_info/node_info.sh b/asterix-events/src/main/resources/events/node_info/node_info.sh
old mode 100755
new mode 100644
diff --git a/asterix-events/src/main/resources/events/node_join/nc_join.sh b/asterix-events/src/main/resources/events/node_join/nc_join.sh
old mode 100755
new mode 100644
diff --git a/asterix-events/src/main/resources/events/node_restart/nc_restart.sh b/asterix-events/src/main/resources/events/node_restart/nc_restart.sh
old mode 100755
new mode 100644
diff --git a/asterix-app/src/test/resources/runtimets/queries/distinct/query-issue443-2/query-issue443-2.1.ddl.aql b/asterix-events/src/main/resources/events/prepare.sh
similarity index 100%
rename from asterix-app/src/test/resources/runtimets/queries/distinct/query-issue443-2/query-issue443-2.1.ddl.aql
rename to asterix-events/src/main/resources/events/prepare.sh
diff --git a/asterix-events/src/main/resources/events/restore/restore.sh b/asterix-events/src/main/resources/events/restore/restore.sh
old mode 100755
new mode 100644
diff --git a/asterix-installer/src/main/resources/schema/installer-conf.xsd b/asterix-events/src/main/resources/schema/installer-conf.xsd
similarity index 100%
rename from asterix-installer/src/main/resources/schema/installer-conf.xsd
rename to asterix-events/src/main/resources/schema/installer-conf.xsd
diff --git a/asterix-events/src/main/resources/scripts/execute.sh b/asterix-events/src/main/resources/scripts/execute.sh
old mode 100755
new mode 100644
diff --git a/asterix-events/src/main/resources/scripts/prepare.sh b/asterix-events/src/main/resources/scripts/prepare.sh
old mode 100755
new mode 100644
diff --git a/asterix-external-data/pom.xml b/asterix-external-data/pom.xml
index f8d5ea2..5917f6b 100644
--- a/asterix-external-data/pom.xml
+++ b/asterix-external-data/pom.xml
@@ -108,6 +108,13 @@
 			<scope>compile</scope>
 		</dependency>
 		<dependency>
+			<groupId>edu.uci.ics.asterix</groupId>
+			<artifactId>asterix-metadata</artifactId>
+			<version>0.8.1-SNAPSHOT</version>
+			<type>jar</type>
+			<scope>compile</scope>
+		</dependency>
+		<dependency>
 			<groupId>com.kenai.nbpwr</groupId>
 			<artifactId>org-apache-commons-io</artifactId>
 			<version>1.3.1-201002241208</version>
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/CNNFeedAdapterFactory.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/CNNFeedAdapterFactory.java
index f1a3d6c..68eac48 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/CNNFeedAdapterFactory.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/CNNFeedAdapterFactory.java
@@ -17,7 +17,8 @@
 import java.util.Map;
 
 import edu.uci.ics.asterix.external.dataset.adapter.CNNFeedAdapter;
-import edu.uci.ics.asterix.external.dataset.adapter.IDatasourceAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IDatasourceAdapter;
+import edu.uci.ics.asterix.metadata.feeds.ITypedDatasetAdapterFactory;
 
 /**
  * A factory class for creating the @see {CNNFeedAdapter}.  
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/HDFSAdapterFactory.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/HDFSAdapterFactory.java
index 4ca3d72..15f702b 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/HDFSAdapterFactory.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/HDFSAdapterFactory.java
@@ -22,7 +22,8 @@
 import org.apache.hadoop.mapred.JobConf;
 
 import edu.uci.ics.asterix.external.dataset.adapter.HDFSAdapter;
-import edu.uci.ics.asterix.external.dataset.adapter.IDatasourceAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IDatasourceAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IGenericDatasetAdapterFactory;
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
 import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/HiveAdapterFactory.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/HiveAdapterFactory.java
index 409eb7a..4fd9411 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/HiveAdapterFactory.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/HiveAdapterFactory.java
@@ -23,7 +23,8 @@
 
 import edu.uci.ics.asterix.external.dataset.adapter.HDFSAdapter;
 import edu.uci.ics.asterix.external.dataset.adapter.HiveAdapter;
-import edu.uci.ics.asterix.external.dataset.adapter.IDatasourceAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IDatasourceAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IGenericDatasetAdapterFactory;
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
 import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java
index e680232..02f04e8 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java
@@ -16,8 +16,9 @@
 
 import java.util.Map;
 
-import edu.uci.ics.asterix.external.dataset.adapter.IDatasourceAdapter;
 import edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IDatasourceAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IGenericDatasetAdapterFactory;
 import edu.uci.ics.asterix.om.types.IAType;
 
 /**
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java
index 232d5e8..bd181e5 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java
@@ -16,8 +16,9 @@
 
 import java.util.Map;
 
-import edu.uci.ics.asterix.external.dataset.adapter.IDatasourceAdapter;
 import edu.uci.ics.asterix.external.dataset.adapter.PullBasedTwitterAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IDatasourceAdapter;
+import edu.uci.ics.asterix.metadata.feeds.ITypedDatasetAdapterFactory;
 
 /**
  * Factory class for creating an instance of PullBasedTwitterAdapter.
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/RSSFeedAdapterFactory.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
index ab18455..6e732ba 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
@@ -16,8 +16,9 @@
 
 import java.util.Map;
 
-import edu.uci.ics.asterix.external.dataset.adapter.IDatasourceAdapter;
 import edu.uci.ics.asterix.external.dataset.adapter.RSSFeedAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IDatasourceAdapter;
+import edu.uci.ics.asterix.metadata.feeds.ITypedDatasetAdapterFactory;
 
 /**
  * Factory class for creating an instance of @see {RSSFeedAdapter}.
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/CNNFeedAdapter.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/CNNFeedAdapter.java
index 9112aae..cfcfd74 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/CNNFeedAdapter.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/CNNFeedAdapter.java
@@ -19,7 +19,8 @@
 import java.util.List;
 import java.util.Map;
 
-import edu.uci.ics.asterix.feed.managed.adapter.IManagedFeedAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IDatasourceAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IManagedFeedAdapter;
 
 /**
  * An Adapter that provides the functionality of fetching news feed from CNN service
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/FileSystemBasedAdapter.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/FileSystemBasedAdapter.java
index 38903ec..6ce18fa 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/FileSystemBasedAdapter.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/FileSystemBasedAdapter.java
@@ -24,6 +24,7 @@
 import edu.uci.ics.asterix.external.util.DNSResolverFactory;
 import edu.uci.ics.asterix.external.util.INodeResolver;
 import edu.uci.ics.asterix.external.util.INodeResolverFactory;
+import edu.uci.ics.asterix.metadata.feeds.AbstractDatasourceAdapter;
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.ATypeTag;
 import edu.uci.ics.asterix.om.types.IAType;
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/HiveAdapter.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/HiveAdapter.java
index 91cf0d2..39484a6 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/HiveAdapter.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/HiveAdapter.java
@@ -16,9 +16,9 @@
 
 import java.util.Map;
 
+import edu.uci.ics.asterix.metadata.feeds.AbstractDatasourceAdapter;
 import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.JobConf;
-
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
 import edu.uci.ics.hyracks.api.comm.IFrameWriter;
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/IFeedClientFactory.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/IFeedClientFactory.java
new file mode 100644
index 0000000..4fb14ff
--- /dev/null
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/IFeedClientFactory.java
@@ -0,0 +1,22 @@
+package edu.uci.ics.asterix.external.dataset.adapter;
+
+import java.util.Map;
+
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.om.types.ARecordType;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+
+public interface IFeedClientFactory {
+
+    public IPullBasedFeedClient createFeedClient(IHyracksTaskContext ctx, Map<String, String> configuration)
+            throws Exception;
+
+    public ARecordType getRecordType() throws AsterixException;
+
+    public FeedClientType getFeedClientType();
+
+    public enum FeedClientType {
+        GENERIC,
+        TYPED
+    }
+}
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/IPullBasedFeedClient.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/IPullBasedFeedClient.java
index aeecb5f..e8aadff 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/IPullBasedFeedClient.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/IPullBasedFeedClient.java
@@ -15,11 +15,18 @@
 package edu.uci.ics.asterix.external.dataset.adapter;
 
 import java.io.DataOutput;
+import java.util.Map;
 
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
 
 public interface IPullBasedFeedClient {
 
+    public enum InflowState {
+        NO_MORE_DATA,
+        DATA_AVAILABLE,
+        DATA_NOT_AVAILABLE
+    }
+
     /**
      * Writes the next fetched tuple into the provided instance of DatatOutput.
      * 
@@ -29,7 +36,7 @@
      *         false if no record was written to the DataOutput instance indicating non-availability of new data.
      * @throws AsterixException
      */
-    public boolean nextTuple(DataOutput dataOutput) throws AsterixException;
+    public InflowState nextTuple(DataOutput dataOutput) throws AsterixException;
 
     /**
      * Provides logic for any corrective action that feed client needs to execute on
@@ -42,10 +49,10 @@
     public void resetOnFailure(Exception e) throws AsterixException;
 
     /**
-     * Terminates a feed, that is data ingestion activity ceases.
-     * 
-     * @throws Exception
+     * @param configuration
      */
-    public void stop() throws Exception;
+    public boolean alter(Map<String, Object> configuration);
+
+    public void stop();
 
 }
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedAdapter.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedAdapter.java
index 93d2057..a789a20 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedAdapter.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedAdapter.java
@@ -15,8 +15,14 @@
 package edu.uci.ics.asterix.external.dataset.adapter;
 
 import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.logging.Logger;
 
-import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.external.dataset.adapter.IPullBasedFeedClient.InflowState;
+import edu.uci.ics.asterix.metadata.feeds.AbstractFeedDatasourceAdapter;
+import edu.uci.ics.asterix.metadata.feeds.FeedPolicyEnforcer;
+import edu.uci.ics.asterix.metadata.feeds.IManagedFeedAdapter;
+import edu.uci.ics.asterix.metadata.feeds.ITypedDatasourceAdapter;
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.hyracks.api.comm.IFrameWriter;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
@@ -29,18 +35,33 @@
  * Captures the common logic for obtaining bytes from an external source
  * and packing them into frames as tuples.
  */
-public abstract class PullBasedAdapter extends AbstractDatasourceAdapter implements ITypedDatasourceAdapter {
+public abstract class PullBasedAdapter extends AbstractFeedDatasourceAdapter implements ITypedDatasourceAdapter,
+        IManagedFeedAdapter {
 
     private static final long serialVersionUID = 1L;
+    private static final Logger LOGGER = Logger.getLogger(PullBasedAdapter.class.getName());
 
     protected ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(1);
     protected IPullBasedFeedClient pullBasedFeedClient;
     protected ARecordType adapterOutputType;
     private FrameTupleAppender appender;
     private ByteBuffer frame;
+    protected boolean continueIngestion = true;
+    protected boolean alterRequested = false;
+    private Map<String, Object> modifiedConfiguration = null;
+    private long tupleCount = 0;
+    private FeedPolicyEnforcer policyEnforcer;
 
     public abstract IPullBasedFeedClient getFeedClient(int partition) throws Exception;
 
+    public long getIngestedRecordsCount() {
+        return tupleCount;
+    }
+
+    public void alter(Map<String, Object> modifedConfiguration) {
+        this.modifiedConfiguration = modifedConfiguration;
+    }
+
     @Override
     public void start(int partition, IFrameWriter writer) throws Exception {
         appender = new FrameTupleAppender(ctx.getFrameSize());
@@ -48,22 +69,42 @@
         appender.reset(frame, true);
 
         pullBasedFeedClient = getFeedClient(partition);
-        boolean moreData = false;
-        while (true) {
+        InflowState inflowState = null;
+        while (continueIngestion) {
             tupleBuilder.reset();
             try {
-                moreData = pullBasedFeedClient.nextTuple(tupleBuilder.getDataOutput());
-                if (moreData) {
-                    tupleBuilder.addFieldEndOffset();
-                    appendTupleToFrame(writer);
-                } else {
-                    FrameUtils.flushFrame(frame, writer);
-                    break;
+                inflowState = pullBasedFeedClient.nextTuple(tupleBuilder.getDataOutput());
+                switch (inflowState) {
+                    case DATA_AVAILABLE:
+                        tupleBuilder.addFieldEndOffset();
+                        appendTupleToFrame(writer);
+                        tupleCount++;
+                        break;
+                    case NO_MORE_DATA:
+                        LOGGER.info("Reached end of feed");
+                        FrameUtils.flushFrame(frame, writer);
+                        continueIngestion = false;
+                        break;
+                    case DATA_NOT_AVAILABLE:
+                        break;
+                }
+                if (alterRequested) {
+                    boolean success = pullBasedFeedClient.alter(modifiedConfiguration);
+                    if (success) {
+                        configuration = modifiedConfiguration;
+                        modifiedConfiguration = null;
+                    }
                 }
             } catch (Exception failureException) {
                 try {
-                    pullBasedFeedClient.resetOnFailure(failureException);
-                    continue;
+                    boolean continueIngestion = policyEnforcer.handleSoftwareFailure(failureException);
+                    if (continueIngestion) {
+                        pullBasedFeedClient.resetOnFailure(failureException);
+                        tupleBuilder.reset();
+                        continue;
+                    } else {
+                        throw failureException;
+                    }
                 } catch (Exception recoveryException) {
                     throw new Exception(recoveryException);
                 }
@@ -71,16 +112,6 @@
         }
     }
 
-    /**
-     * Allows an adapter to handle a runtime exception.
-     * @param e exception encountered during runtime
-     * @throws AsterixException
-     */
-    public void resetOnFailure(Exception e) throws AsterixException {
-        pullBasedFeedClient.resetOnFailure(e);
-        tupleBuilder.reset();
-    }
-
     private void appendTupleToFrame(IFrameWriter writer) throws HyracksDataException {
         if (!appender.append(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray(), 0, tupleBuilder.getSize())) {
             FrameUtils.flushFrame(frame, writer);
@@ -97,4 +128,15 @@
         return adapterOutputType;
     }
 
+    /**
+     * Discontinue the ingestion of data and end the feed.
+     * 
+     * @throws Exception
+     */
+    public void stop() throws Exception {
+        continueIngestion = false;
+        dumpStatistics();
+        timer.cancel();
+    }
+
 }
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedFeedClient.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedFeedClient.java
index 6d05bde..a5cb037 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedFeedClient.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedFeedClient.java
@@ -17,46 +17,138 @@
 import java.io.DataOutput;
 import java.io.IOException;
 
+import edu.uci.ics.asterix.builders.IARecordBuilder;
+import edu.uci.ics.asterix.builders.RecordBuilder;
+import edu.uci.ics.asterix.builders.UnorderedListBuilder;
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
 import edu.uci.ics.asterix.dataflow.data.nontagged.serde.ARecordSerializerDeserializer;
+import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import edu.uci.ics.asterix.om.base.ABoolean;
+import edu.uci.ics.asterix.om.base.AInt32;
+import edu.uci.ics.asterix.om.base.AMutableDateTime;
+import edu.uci.ics.asterix.om.base.AMutableInt32;
+import edu.uci.ics.asterix.om.base.AMutablePoint;
 import edu.uci.ics.asterix.om.base.AMutableRecord;
+import edu.uci.ics.asterix.om.base.AMutableString;
+import edu.uci.ics.asterix.om.base.AMutableUnorderedList;
+import edu.uci.ics.asterix.om.base.AString;
+import edu.uci.ics.asterix.om.base.IACursor;
+import edu.uci.ics.asterix.om.base.IAObject;
+import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.AUnorderedListType;
+import edu.uci.ics.asterix.om.types.BuiltinType;
 import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
 
 public abstract class PullBasedFeedClient implements IPullBasedFeedClient {
 
     protected ARecordSerializerDeserializer recordSerDe;
     protected AMutableRecord mutableRecord;
     protected boolean messageReceived;
-    protected boolean continueIngestion=true;
+    protected boolean continueIngestion = true;
+    protected IARecordBuilder recordBuilder = new RecordBuilder();
 
-    public abstract boolean setNextRecord() throws Exception;
+    protected AMutableString aString = new AMutableString("");
+    protected AMutableInt32 aInt32 = new AMutableInt32(0);
+    protected AMutablePoint aPoint = new AMutablePoint(0, 0);
+    protected AMutableDateTime aDateTime = new AMutableDateTime(0);
+
+    @SuppressWarnings("unchecked")
+    protected ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
+            .getSerializerDeserializer(BuiltinType.ASTRING);
+    @SuppressWarnings("unchecked")
+    protected ISerializerDeserializer<ABoolean> booleanSerde = AqlSerializerDeserializerProvider.INSTANCE
+            .getSerializerDeserializer(BuiltinType.ABOOLEAN);
+    @SuppressWarnings("unchecked")
+    protected ISerializerDeserializer<AInt32> int32Serde = AqlSerializerDeserializerProvider.INSTANCE
+            .getSerializerDeserializer(BuiltinType.AINT32);
+
+    public abstract InflowState setNextRecord() throws Exception;
 
     @Override
-    public boolean nextTuple(DataOutput dataOutput) throws AsterixException {
+    public InflowState nextTuple(DataOutput dataOutput) throws AsterixException {
         try {
-            boolean newData = setNextRecord();
-            if (newData && continueIngestion) {
-                IAType t = mutableRecord.getType();
-                ATypeTag tag = t.getTypeTag();
-                try {
+            InflowState state = setNextRecord();
+            boolean first = true;
+            switch (state) {
+                case DATA_AVAILABLE:
+                    IAType t = mutableRecord.getType();
+                    ATypeTag tag = t.getTypeTag();
                     dataOutput.writeByte(tag.serialize());
-                } catch (IOException e) {
-                    throw new HyracksDataException(e);
-                }
-                recordSerDe.serialize(mutableRecord, dataOutput);
-                return true;
+                    if (first) {
+                        recordBuilder.reset(mutableRecord.getType());
+                        first = false;
+                    }
+                    recordBuilder.init();
+                    writeRecord(mutableRecord, dataOutput, recordBuilder);
+                    break;
+ 
+                case DATA_NOT_AVAILABLE:
+                    break;
+                case NO_MORE_DATA:
+                    break;
             }
-            return false;
+            return state;
         } catch (Exception e) {
             throw new AsterixException(e);
         }
 
     }
 
-    @Override
-    public void stop() {
-        continueIngestion = false;
+    @SuppressWarnings("unchecked")
+    private void writeRecord(AMutableRecord record, DataOutput dataOutput, IARecordBuilder recordBuilder)
+            throws IOException, AsterixException {
+        ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
+        int numFields = record.getType().getFieldNames().length;
+        for (int pos = 0; pos < numFields; pos++) {
+            fieldValue.reset();
+            IAObject obj = record.getValueByPos(pos);
+            writeObject(obj, fieldValue.getDataOutput());
+            recordBuilder.addField(pos, fieldValue);
+        }
+        recordBuilder.write(dataOutput, false);
+    }
+
+    private void writeObject(IAObject obj, DataOutput dataOutput) throws IOException, AsterixException {
+        switch (obj.getType().getTypeTag()) {
+            case RECORD:
+                ATypeTag tag = obj.getType().getTypeTag();
+                try {
+                    dataOutput.writeByte(tag.serialize());
+                } catch (IOException e) {
+                    throw new HyracksDataException(e);
+                }
+                IARecordBuilder recordBuilder = new RecordBuilder();
+                recordBuilder.reset((ARecordType) obj.getType());
+                recordBuilder.init();
+                writeRecord((AMutableRecord) obj, dataOutput, recordBuilder);
+                break;
+            case UNORDEREDLIST:
+                tag = obj.getType().getTypeTag();
+                try {
+                    dataOutput.writeByte(tag.serialize());
+                } catch (IOException e) {
+                    throw new HyracksDataException(e);
+                }
+                UnorderedListBuilder listBuilder = new UnorderedListBuilder();
+                listBuilder.reset((AUnorderedListType) ((AMutableUnorderedList) obj).getType());
+                IACursor cursor = ((AMutableUnorderedList) obj).getCursor();
+                ArrayBackedValueStorage listItemValue = new ArrayBackedValueStorage();
+                while (cursor.next()) {
+                    listItemValue.reset();
+                    IAObject item = cursor.get();
+                    writeObject(item, listItemValue.getDataOutput());
+                    listBuilder.addItem(listItemValue);
+                }
+                listBuilder.write(dataOutput, false);
+                break;
+            default:
+                AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(obj.getType()).serialize(obj,
+                        dataOutput);
+                break;
+        }
     }
 }
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedTwitterAdapter.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedTwitterAdapter.java
index 2715a00..97b1d77 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedTwitterAdapter.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedTwitterAdapter.java
@@ -14,10 +14,9 @@
  */
 package edu.uci.ics.asterix.external.dataset.adapter;
 
-import java.util.HashMap;
 import java.util.Map;
 
-import edu.uci.ics.asterix.feed.managed.adapter.IManagedFeedAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IManagedFeedAdapter;
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.BuiltinType;
 import edu.uci.ics.asterix.om.types.IAType;
@@ -36,8 +35,6 @@
     public static final String QUERY = "query";
     public static final String INTERVAL = "interval";
 
-    private boolean alterRequested = false;
-    private Map<String, String> alteredParams = new HashMap<String, String>();
     private ARecordType recordType;
 
     private PullBasedTwitterFeedClient tweetClient;
@@ -73,25 +70,18 @@
     }
 
     @Override
-    public void alter(Map<String, String> properties) {
+    public void alter(Map<String, Object> properties) {
         alterRequested = true;
-        this.alteredParams = properties;
     }
 
     public boolean isAlterRequested() {
         return alterRequested;
     }
 
-    public Map<String, String> getAlteredParams() {
-        return alteredParams;
-    }
-
     public void postAlteration() {
-        alteredParams = null;
         alterRequested = false;
     }
 
-    @Override
     public ARecordType getAdapterOutputType() {
         return recordType;
     }
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedTwitterFeedClient.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedTwitterFeedClient.java
index 9f44a03..b84e357 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedTwitterFeedClient.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedTwitterFeedClient.java
@@ -88,11 +88,11 @@
     }
 
     @Override
-    public boolean setNextRecord() throws Exception {
+    public InflowState setNextRecord() throws Exception {
         Tweet tweet;
         tweet = getNextTweet();
         if (tweet == null) {
-            return false;
+            return InflowState.DATA_NOT_AVAILABLE;
         }
         int numFields = recordType.getFieldNames().length;
 
@@ -106,7 +106,7 @@
             mutableRecord.setValueAtPos(i, mutableFields[i]);
         }
         id++;
-        return true;
+        return InflowState.DATA_AVAILABLE;
     }
 
     @Override
@@ -114,4 +114,16 @@
         // TOOO: implement resetting logic for Twitter
     }
 
+    @Override
+    public boolean alter(Map<String, Object> configuration) {
+        // TODO Auto-generated method stub
+        return false;
+    }
+
+    @Override
+    public void stop() {
+        // TODO Auto-generated method stub
+        
+    }
+
 }
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/RSSFeedAdapter.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/RSSFeedAdapter.java
index 8379b18..c3c5ae0 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/RSSFeedAdapter.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/RSSFeedAdapter.java
@@ -19,7 +19,7 @@
 import java.util.List;
 import java.util.Map;
 
-import edu.uci.ics.asterix.feed.managed.adapter.IManagedFeedAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IManagedFeedAdapter;
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.BuiltinType;
 import edu.uci.ics.asterix.om.types.IAType;
@@ -37,7 +37,7 @@
     private List<String> feedURLs = new ArrayList<String>();
     private boolean isStopRequested = false;
     private boolean isAlterRequested = false;
-    private Map<String, String> alteredParams = new HashMap<String, String>();
+    private Map<String, Object> alteredParams = new HashMap<String, Object>();
     private String id_prefix = "";
     private ARecordType recordType;
 
@@ -55,7 +55,7 @@
     }
 
     @Override
-    public void alter(Map<String, String> properties) {
+    public void alter(Map<String, Object> properties) {
         isAlterRequested = true;
         this.alteredParams = properties;
         reconfigure(properties);
@@ -93,7 +93,7 @@
         }
     }
 
-    protected void reconfigure(Map<String, String> arguments) {
+    protected void reconfigure(Map<String, Object> arguments) {
         String rssURLProperty = (String) configuration.get(KEY_RSS_URL);
         if (rssURLProperty != null) {
             initializeFeedURLs(rssURLProperty);
@@ -114,7 +114,7 @@
         return isAlterRequested;
     }
 
-    public Map<String, String> getAlteredParams() {
+    public Map<String, Object> getAlteredParams() {
         return alteredParams;
     }
 
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/RSSFeedClient.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/RSSFeedClient.java
index d89674f..531eb3d 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/RSSFeedClient.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/RSSFeedClient.java
@@ -18,6 +18,7 @@
 import java.net.URL;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Queue;
 
 import com.sun.syndication.feed.synd.SyndEntryImpl;
@@ -81,10 +82,10 @@
     }
 
     @Override
-    public boolean setNextRecord() throws Exception {
+    public InflowState setNextRecord() throws Exception {
         SyndEntryImpl feedEntry = getNextRSSFeed();
         if (feedEntry == null) {
-            return false;
+            return InflowState.DATA_NOT_AVAILABLE;
         }
         tupleFieldValues[0] = idPrefix + ":" + id;
         tupleFieldValues[1] = feedEntry.getTitle();
@@ -96,7 +97,7 @@
             mutableRecord.setValueAtPos(i, mutableFields[i]);
         }
         id++;
-        return true;
+        return InflowState.DATA_AVAILABLE;
     }
 
     private SyndEntryImpl getNextRSSFeed() throws Exception {
@@ -138,6 +139,18 @@
 
     }
 
+    @Override
+    public boolean alter(Map<String, Object> configuration) {
+        // TODO Auto-generated method stub
+        return false;
+    }
+
+    @Override
+    public void stop() {
+        // TODO Auto-generated method stub
+
+    }
+
 }
 
 class FetcherEventListenerImpl implements FetcherListener {
diff --git a/asterix-external-data/src/main/resources/feed_policy/basic.pol b/asterix-external-data/src/main/resources/feed_policy/basic.pol
new file mode 100644
index 0000000..8aed9f8
--- /dev/null
+++ b/asterix-external-data/src/main/resources/feed_policy/basic.pol
@@ -0,0 +1,8 @@
+feed.policy.name=Basic
+
+software.failure.persist.exception=false
+software.failure.continue.on.failure=false
+hardware.failure.auto.restart=false
+
+statistics.collect=false
+elastic=false
diff --git a/asterix-external-data/src/main/resources/feed_policy/basic_monitored.pol b/asterix-external-data/src/main/resources/feed_policy/basic_monitored.pol
new file mode 100644
index 0000000..efca65e
--- /dev/null
+++ b/asterix-external-data/src/main/resources/feed_policy/basic_monitored.pol
@@ -0,0 +1,11 @@
+feed.policy.name=Basic_Monitored
+
+software.failure.persist.exception=true
+software.failure.continue.on.failure=false
+hardware.failure.auto.restart=false
+
+statistics.collect=true
+statistics.collect.period=60
+statistics.collect.period.unit=sec
+
+elastic=false
diff --git a/asterix-external-data/src/main/resources/feed_policy/mission_critical.pol b/asterix-external-data/src/main/resources/feed_policy/mission_critical.pol
new file mode 100644
index 0000000..0a1da08
--- /dev/null
+++ b/asterix-external-data/src/main/resources/feed_policy/mission_critical.pol
@@ -0,0 +1,17 @@
+feed.policy.name=Mission_Critical
+
+software.failure.persist.exception=true
+software.failure.continue.on.failure=true
+hardware.failure.auto.restart=true
+
+statistics.collect=true
+statistics.collect.period=60
+statistics.collect.period.unit=sec
+
+elastic=true
+criterion.ingestion.rate.percent=true
+criterion.ingestion.rate.percent.delta=50
+
+criterion.ingestion.rate.absolute=true
+criterion.ingestion.rate.absolute.delta=2000
+criterion.ingestion.rate.absolute.unit=sec
diff --git a/asterix-external-data/src/main/resources/feed_policy/twitter.pol b/asterix-external-data/src/main/resources/feed_policy/twitter.pol
new file mode 100644
index 0000000..5fa507f
--- /dev/null
+++ b/asterix-external-data/src/main/resources/feed_policy/twitter.pol
@@ -0,0 +1,17 @@
+feed.policy.name=Twitter
+
+software.failure.persist.exception=true
+software.failure.continue.on.failure=true
+hardware.failure.auto.restart=false
+
+statistics.collect=true
+statistics.collect.period=60
+statistics.collect.period.unit=sec
+
+elastic=true
+criterion.ingestion.rate.percent=true
+criterion.ingestion.rate.percent.delta=50
+
+criterion.ingestion.rate.absolute=true
+criterion.ingestion.rate.absolute.delta=2000
+criterion.ingestion.rate.absolute.unit=sec
diff --git a/asterix-installer/pom.xml b/asterix-installer/pom.xml
index 242f873..f06238d 100644
--- a/asterix-installer/pom.xml
+++ b/asterix-installer/pom.xml
@@ -35,65 +35,6 @@
 					<target>1.6</target>
 				</configuration>
 			</plugin>
-
-			<plugin>
-				<groupId>org.jvnet.jaxb2.maven2</groupId>
-				<artifactId>maven-jaxb2-plugin</artifactId>
-				<executions>
-					<execution>
-						<id>configuration</id>
-						<goals>
-							<goal>generate</goal>
-						</goals>
-						<configuration>
-							<args>
-								<arg>-Xsetters</arg>
-								<arg>-Xvalue-constructor</arg>
-							</args>
-							<plugins>
-								<plugin>
-									<groupId>org.jvnet.jaxb2_commons</groupId>
-									<artifactId>jaxb2-basics</artifactId>
-									<version>0.6.2</version>
-								</plugin>
-								<plugin>
-									<groupId>org.jvnet.jaxb2_commons</groupId>
-									<artifactId>jaxb2-value-constructor</artifactId>
-									<version>3.0</version>
-								</plugin>
-							</plugins>
-							<schemaDirectory>src/main/resources/schema</schemaDirectory>
-							<schemaIncludes>
-								<include>installer-conf.xsd</include>
-							</schemaIncludes>
-							<generatePackage>edu.uci.ics.asterix.installer.schema.conf</generatePackage>
-							<generateDirectory>${project.build.directory}/generated-sources/configuration</generateDirectory>
-						</configuration>
-					</execution>
-					<execution>
-						<id>cluster</id>
-						<goals>
-							<goal>generate</goal>
-						</goals>
-						<configuration>
-							<args>
-								<arg>-Xsetters</arg>
-								<arg>-Xvalue-constructor</arg>
-							</args>
-							<schemaDirectory>src/main/resources/schema</schemaDirectory>
-							<schemaIncludes>
-								<include>cluster.xsd</include>
-							</schemaIncludes>
-							<generatePackage>edu.uci.ics.asterix.installer.schema.cluster</generatePackage>
-							<bindingDirectory>src/main/resources/schema</bindingDirectory>
-							<bindingIncludes>
-								<bindingInclude>jaxb-bindings.xjb</bindingInclude>
-							</bindingIncludes>
-							<generateDirectory>${project.build.directory}/generated-sources/cluster</generateDirectory>
-						</configuration>
-					</execution>
-				</executions>
-			</plugin>
 			<plugin>
 				<artifactId>maven-assembly-plugin</artifactId>
 				<version>2.2-beta-5</version>
diff --git a/asterix-installer/src/main/assembly/binary-assembly.xml b/asterix-installer/src/main/assembly/binary-assembly.xml
index d42c5ae..f21e7b9c 100644
--- a/asterix-installer/src/main/assembly/binary-assembly.xml
+++ b/asterix-installer/src/main/assembly/binary-assembly.xml
@@ -13,112 +13,108 @@
  ! limitations under the License.
  !-->
 <assembly>
-  <id>binary-assembly</id>
-  <formats>
-    <format>zip</format>
-    <format>dir</format>
-  </formats>
-  <includeBaseDirectory>false</includeBaseDirectory>
-  <fileSets>
-    <fileSet>
-      <directory>src/main/resources/conf</directory>
-      <outputDirectory>conf</outputDirectory>
-    </fileSet>
-    <fileSet>
-      <directory>src/main/resources/clusters</directory>
-      <outputDirectory>clusters</outputDirectory>
-    </fileSet>
-    <fileSet>
-      <directory>src/main/resources/zookeeper</directory>
-      <fileMode>0755</fileMode>
-      <outputDirectory>.installer/zookeeper/bin</outputDirectory>
-    </fileSet>
-    <fileSet>
-      <directory>src/main/resources/scripts</directory>
-      <fileMode>0755</fileMode>
-      <includes>
-        <include>managix</include>
-      </includes>
-      <outputDirectory>bin</outputDirectory>
-    </fileSet>
-    <fileSet>
-      <directory>src/main/resources/scripts</directory>
-      <fileMode>0755</fileMode>
-      <excludes>
-        <exclude>managix</exclude>
-      </excludes>
-      <outputDirectory>.installer/scripts</outputDirectory>
-    </fileSet>
-    <fileSet>
-      <directory>src/main/resources/hadoop-0.20.2</directory>
-      <outputDirectory>.installer/hadoop-0.20.2</outputDirectory>
-      <fileMode>0755</fileMode>
-    </fileSet>
-    <fileSet>
-      <directory>target</directory>
-      <outputDirectory>lib</outputDirectory>
-      <includes>
-        <include>*.jar</include>
-      </includes>
-    </fileSet>
-    <fileSet>
-      <directory>../asterix-events/src/main/resources/events</directory>
-      <outputDirectory>.installer/eventrix/events</outputDirectory>
-      <fileMode>0755</fileMode>
-    </fileSet>
-    <fileSet>
-      <directory>../asterix-events/src/main/resources/scripts</directory>
-      <outputDirectory>.installer/eventrix/scripts</outputDirectory>
-    </fileSet>
-  </fileSets>
-  <dependencySets>
-    <dependencySet>
-      <includes>
-         <include>log4j:log4j</include>
-         <include>edu.uci.ics.asterix:asterix-events</include>
-         <include>edu.uci.ics.asterix:asterix-common</include>
-         <include>org.apache.zookeeper:zookeeper</include>
-         <include>args4j:args4j</include>
-         <include>log4j:log4j</include>
-         <include>commons-io:commons-io</include>
-         <include>org.slf4j:slf4j-api</include>
-         <include>org.slf4j:slf4j-log4j12</include>
-       </includes>
-       <unpack>false</unpack>
-       <outputDirectory>lib</outputDirectory>
-    </dependencySet>
-    <dependencySet>
-      <includes>
-        <include>org.apache.hadoop:hadoop-core</include>
-        <include>commons-cli:commons-cli</include>
-        <include>commons-logging:commons-logging</include>
-      </includes>
-      <unpack>false</unpack>
-      <outputDirectory>.installer/hadoop-0.20.2/lib</outputDirectory>
-    </dependencySet>
-    <dependencySet>
-      <includes>
-        <include>org.apache.zookeeper:zookeeper</include>
-        <include>log4j:log4j</include>
-        <include>org.slf4j:slf4j-api</include>
-      </includes>
-      <unpack>false</unpack>
-      <outputDirectory>.installer/zookeeper/lib</outputDirectory>
-    </dependencySet>
-    <dependencySet>
-      <outputDirectory>asterix</outputDirectory>
-      <includes>
-        <include>asterix-server*</include>
-      </includes>
-      <useTransitiveDependencies>false</useTransitiveDependencies>
-    </dependencySet>
-    <dependencySet>
-      <outputDirectory>.installer/eventrix</outputDirectory>
-      <includes>
-        <include>asterix-events*</include>
-      </includes>
-      <unpack>false</unpack>
-      <useTransitiveDependencies>false</useTransitiveDependencies>
-    </dependencySet>
-  </dependencySets>
+	<id>binary-assembly</id>
+	<formats>
+		<format>zip</format>
+		<format>dir</format>
+	</formats>
+	<includeBaseDirectory>false</includeBaseDirectory>
+	<fileSets>
+		<fileSet>
+			<directory>src/main/resources/conf</directory>
+			<outputDirectory>conf</outputDirectory>
+		</fileSet>
+		<fileSet>
+			<directory>src/main/resources/clusters</directory>
+			<outputDirectory>clusters</outputDirectory>
+		</fileSet>
+		<fileSet>
+			<directory>src/main/resources/zookeeper</directory>
+			<fileMode>0755</fileMode>
+			<outputDirectory>.installer/zookeeper/bin</outputDirectory>
+		</fileSet>
+		<fileSet>
+			<directory>src/main/resources/scripts</directory>
+			<fileMode>0755</fileMode>
+			<includes>
+				<include>managix</include>
+			</includes>
+			<outputDirectory>bin</outputDirectory>
+		</fileSet>
+		<fileSet>
+			<directory>src/main/resources/scripts</directory>
+			<fileMode>0755</fileMode>
+			<excludes>
+				<exclude>managix</exclude>
+			</excludes>
+			<outputDirectory>.installer/scripts</outputDirectory>
+		</fileSet>
+		<fileSet>
+			<directory>src/main/resources/hadoop-0.20.2</directory>
+			<outputDirectory>.installer/hadoop-0.20.2</outputDirectory>
+			<fileMode>0755</fileMode>
+		</fileSet>
+		<fileSet>
+			<directory>target</directory>
+			<outputDirectory>lib</outputDirectory>
+			<includes>
+				<include>*.jar</include>
+			</includes>
+		</fileSet>
+		<fileSet>
+			<directory>../asterix-events/src/main/resources/events</directory>
+			<outputDirectory>.installer/events</outputDirectory>
+			<fileMode>0755</fileMode>
+		</fileSet>
+	</fileSets>
+	<dependencySets>
+		<dependencySet>
+			<includes>
+				<include>log4j:log4j</include>
+				<include>edu.uci.ics.asterix:asterix-events</include>
+				<include>edu.uci.ics.asterix:asterix-common</include>
+				<include>org.apache.zookeeper:zookeeper</include>
+				<include>args4j:args4j</include>
+				<include>log4j:log4j</include>
+				<include>commons-io:commons-io</include>
+				<include>org.slf4j:slf4j-api</include>
+				<include>org.slf4j:slf4j-log4j12</include>
+			</includes>
+			<unpack>false</unpack>
+			<outputDirectory>lib</outputDirectory>
+		</dependencySet>
+		<dependencySet>
+			<includes>
+				<include>org.apache.hadoop:hadoop-core</include>
+				<include>commons-cli:commons-cli</include>
+				<include>commons-logging:commons-logging</include>
+			</includes>
+			<unpack>false</unpack>
+			<outputDirectory>.installer/hadoop-0.20.2/lib</outputDirectory>
+		</dependencySet>
+		<dependencySet>
+			<includes>
+				<include>org.apache.zookeeper:zookeeper</include>
+				<include>log4j:log4j</include>
+				<include>org.slf4j:slf4j-api</include>
+			</includes>
+			<unpack>false</unpack>
+			<outputDirectory>.installer/zookeeper/lib</outputDirectory>
+		</dependencySet>
+		<dependencySet>
+			<outputDirectory>asterix</outputDirectory>
+			<includes>
+				<include>asterix-server*</include>
+			</includes>
+			<useTransitiveDependencies>false</useTransitiveDependencies>
+		</dependencySet>
+		<dependencySet>
+			<outputDirectory>.installer/events</outputDirectory>
+			<includes>
+				<include>asterix-events*</include>
+			</includes>
+			<unpack>false</unpack>
+			<useTransitiveDependencies>false</useTransitiveDependencies>
+		</dependencySet>
+	</dependencySets>
 </assembly>
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/AlterCommand.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/AlterCommand.java
index 8e4b2bc..4be8cd5 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/AlterCommand.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/AlterCommand.java
@@ -19,53 +19,60 @@
 import org.kohsuke.args4j.Option;
 
 import edu.uci.ics.asterix.common.configuration.AsterixConfiguration;
+import edu.uci.ics.asterix.event.model.AsterixInstance;
+import edu.uci.ics.asterix.event.model.AsterixInstance.State;
+import edu.uci.ics.asterix.event.service.AsterixEventServiceUtil;
+import edu.uci.ics.asterix.event.service.ILookupService;
+import edu.uci.ics.asterix.event.service.ServiceProvider;
 import edu.uci.ics.asterix.installer.driver.InstallerDriver;
 import edu.uci.ics.asterix.installer.driver.InstallerUtil;
-import edu.uci.ics.asterix.installer.model.AsterixInstance;
-import edu.uci.ics.asterix.installer.model.AsterixInstance.State;
-import edu.uci.ics.asterix.installer.service.ILookupService;
-import edu.uci.ics.asterix.installer.service.ServiceProvider;
 
 public class AlterCommand extends AbstractCommand {
 
-    @Override
-    protected void execCommand() throws Exception {
-        InstallerDriver.initConfig(true);
-        String instanceName = ((AlterConfig) config).name;
-        InstallerUtil.validateAsterixInstanceExists(instanceName, State.INACTIVE);
-        ILookupService lookupService = ServiceProvider.INSTANCE.getLookupService();
-        AsterixInstance instance = lookupService.getAsterixInstance(instanceName);
-        InstallerUtil.createClusterProperties(instance.getCluster(), instance.getAsterixConfiguration());
-        AsterixConfiguration asterixConfiguration = InstallerUtil
-                .getAsterixConfiguration(((AlterConfig) config).confPath);
-        instance.setAsterixConfiguration(asterixConfiguration);
-        instance.setModifiedTimestamp(new Date());
-        lookupService.updateAsterixInstance(instance);
-        LOGGER.info("Altered configuration settings for Asterix instance: " + instanceName);
+	@Override
+	protected void execCommand() throws Exception {
+		InstallerDriver.initConfig(true);
+		String instanceName = ((AlterConfig) config).name;
+		AsterixEventServiceUtil.validateAsterixInstanceExists(instanceName,
+				State.INACTIVE);
+		ILookupService lookupService = ServiceProvider.INSTANCE
+				.getLookupService();
+		AsterixInstance instance = ServiceProvider.INSTANCE.getLookupService()
+				.getAsterixInstance(instanceName);
+		AsterixEventServiceUtil.createClusterProperties(instance.getCluster(),
+				instance.getAsterixConfiguration());
+		AsterixConfiguration asterixConfiguration = InstallerUtil
+				.getAsterixConfiguration(((AlterConfig) config).confPath);
+		instance.setAsterixConfiguration(asterixConfiguration);
+		instance.setModifiedTimestamp(new Date());
+		lookupService.updateAsterixInstance(instance);
+		LOGGER.info("Altered configuration settings for Asterix instance: "
+				+ instanceName);
 
-    }
+	}
 
-    @Override
-    protected CommandConfig getCommandConfig() {
-        return new AlterConfig();
-    }
+	@Override
+	protected CommandConfig getCommandConfig() {
+		return new AlterConfig();
+	}
 
-    @Override
-    protected String getUsageDescription() {
-        return "\nAlter the instance's configuration settings."
-                + "\nPrior to running this command, the instance is required to be INACTIVE state."
-                + "\nChanged configuration settings will be reflected when the instance is started."
-                + "\n\nAvailable arguments/options" + "\n-n name of the ASTERIX instance.";
-    }
+	@Override
+	protected String getUsageDescription() {
+		return "\nAlter the instance's configuration settings."
+				+ "\nPrior to running this command, the instance is required to be INACTIVE state."
+				+ "\nChanged configuration settings will be reflected when the instance is started."
+				+ "\n\nAvailable arguments/options"
+				+ "\n-n name of the ASTERIX instance.";
+	}
 
 }
 
 class AlterConfig extends CommandConfig {
 
-    @Option(name = "-n", required = true, usage = "Name of Asterix Instance")
-    public String name;
+	@Option(name = "-n", required = true, usage = "Name of Asterix Instance")
+	public String name;
 
-    @Option(name = "-a", required = true, usage = "Path to asterix instance configuration")
-    public String confPath;
+	@Option(name = "-a", required = true, usage = "Path to asterix instance configuration")
+	public String confPath;
 
 }
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/BackupCommand.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/BackupCommand.java
index 91cbc86..3f30619 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/BackupCommand.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/BackupCommand.java
@@ -19,15 +19,16 @@
 
 import org.kohsuke.args4j.Option;
 
+import edu.uci.ics.asterix.event.model.AsterixInstance;
+import edu.uci.ics.asterix.event.model.AsterixInstance.State;
+import edu.uci.ics.asterix.event.model.BackupInfo;
 import edu.uci.ics.asterix.event.schema.pattern.Patterns;
+import edu.uci.ics.asterix.event.service.AsterixEventService;
+import edu.uci.ics.asterix.event.service.AsterixEventServiceUtil;
+import edu.uci.ics.asterix.event.service.ServiceProvider;
+import edu.uci.ics.asterix.event.util.PatternCreator;
 import edu.uci.ics.asterix.installer.driver.InstallerDriver;
-import edu.uci.ics.asterix.installer.driver.InstallerUtil;
-import edu.uci.ics.asterix.installer.events.PatternCreator;
-import edu.uci.ics.asterix.installer.model.AsterixInstance;
-import edu.uci.ics.asterix.installer.model.AsterixInstance.State;
-import edu.uci.ics.asterix.installer.model.BackupInfo;
 import edu.uci.ics.asterix.installer.schema.conf.Backup;
-import edu.uci.ics.asterix.installer.service.ServiceProvider;
 
 public class BackupCommand extends AbstractCommand {
 
@@ -37,12 +38,12 @@
     protected void execCommand() throws Exception {
         InstallerDriver.initConfig(true);
         String asterixInstanceName = ((BackupConfig) config).name;
-        AsterixInstance instance = InstallerUtil.validateAsterixInstanceExists(asterixInstanceName, State.INACTIVE);
+        AsterixInstance instance = AsterixEventServiceUtil.validateAsterixInstanceExists(asterixInstanceName,
+                State.INACTIVE);
         List<BackupInfo> backupInfo = instance.getBackupInfo();
-        PatternCreator pc = new PatternCreator();
-        Backup backupConf = InstallerDriver.getConfiguration().getBackup();
-        Patterns patterns = pc.getBackUpAsterixPattern(instance, backupConf);
-        InstallerUtil.getEventrixClient(instance.getCluster()).submit(patterns);
+        Backup backupConf = AsterixEventService.getConfiguration().getBackup();
+        Patterns patterns = PatternCreator.INSTANCE.getBackUpAsterixPattern(instance, backupConf);
+        AsterixEventService.getAsterixEventServiceClient(instance.getCluster()).submit(patterns);
         int backupId = backupInfo.size();
         BackupInfo binfo = new BackupInfo(backupId, new Date(), backupConf);
         backupInfo.add(binfo);
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/CreateCommand.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/CreateCommand.java
index 2803fee..7784d79 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/CreateCommand.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/CreateCommand.java
@@ -19,17 +19,19 @@
 import org.kohsuke.args4j.Option;
 
 import edu.uci.ics.asterix.common.configuration.AsterixConfiguration;
+import edu.uci.ics.asterix.event.error.VerificationUtil;
+import edu.uci.ics.asterix.event.management.AsterixEventServiceClient;
 import edu.uci.ics.asterix.event.management.EventUtil;
-import edu.uci.ics.asterix.event.management.EventrixClient;
+import edu.uci.ics.asterix.event.model.AsterixInstance;
+import edu.uci.ics.asterix.event.model.AsterixRuntimeState;
 import edu.uci.ics.asterix.event.schema.cluster.Cluster;
 import edu.uci.ics.asterix.event.schema.pattern.Patterns;
+import edu.uci.ics.asterix.event.service.AsterixEventService;
+import edu.uci.ics.asterix.event.service.AsterixEventServiceUtil;
+import edu.uci.ics.asterix.event.service.ServiceProvider;
+import edu.uci.ics.asterix.event.util.PatternCreator;
 import edu.uci.ics.asterix.installer.driver.InstallerDriver;
 import edu.uci.ics.asterix.installer.driver.InstallerUtil;
-import edu.uci.ics.asterix.installer.error.VerificationUtil;
-import edu.uci.ics.asterix.installer.events.PatternCreator;
-import edu.uci.ics.asterix.installer.model.AsterixInstance;
-import edu.uci.ics.asterix.installer.model.AsterixRuntimeState;
-import edu.uci.ics.asterix.installer.service.ServiceProvider;
 
 public class CreateCommand extends AbstractCommand {
 
@@ -46,29 +48,29 @@
             throw new Exception("Cannot create an Asterix instance.");
         }
         asterixInstanceName = ((CreateConfig) config).name;
-        InstallerUtil.validateAsterixInstanceNotExists(asterixInstanceName);
+        AsterixEventServiceUtil.validateAsterixInstanceNotExists(asterixInstanceName);
         CreateConfig createConfig = (CreateConfig) config;
         cluster = EventUtil.getCluster(createConfig.clusterPath);
         asterixConfiguration = InstallerUtil.getAsterixConfiguration(createConfig.asterixConfPath);
-        AsterixInstance asterixInstance = InstallerUtil.createAsterixInstance(asterixInstanceName, cluster,
+        AsterixInstance asterixInstance = AsterixEventServiceUtil.createAsterixInstance(asterixInstanceName, cluster,
                 asterixConfiguration);
-        InstallerUtil.evaluateConflictWithOtherInstances(asterixInstance);
-        InstallerUtil.createAsterixZip(asterixInstance);
-        InstallerUtil.createClusterProperties(cluster, asterixConfiguration);
-        EventrixClient eventrixClient = InstallerUtil.getEventrixClient(cluster);
-        PatternCreator pc = new PatternCreator();
+        AsterixEventServiceUtil.evaluateConflictWithOtherInstances(asterixInstance);
+        AsterixEventServiceUtil.createAsterixZip(asterixInstance);
+        AsterixEventServiceUtil.createClusterProperties(cluster, asterixConfiguration);
+        AsterixEventServiceClient eventrixClient = AsterixEventService.getAsterixEventServiceClient(cluster, true, false);
 
-        Patterns asterixBinarytrasnferPattern = pc.getAsterixBinaryTransferPattern(asterixInstanceName, cluster);
+        Patterns asterixBinarytrasnferPattern = PatternCreator.INSTANCE.getAsterixBinaryTransferPattern(
+                asterixInstanceName, cluster);
         eventrixClient.submit(asterixBinarytrasnferPattern);
 
-        Patterns patterns = pc.getStartAsterixPattern(asterixInstanceName, cluster);
+        Patterns patterns = PatternCreator.INSTANCE.getStartAsterixPattern(asterixInstanceName, cluster);
         eventrixClient.submit(patterns);
 
         AsterixRuntimeState runtimeState = VerificationUtil.getAsterixRuntimeState(asterixInstance);
         VerificationUtil.updateInstanceWithRuntimeDescription(asterixInstance, runtimeState, true);
         ServiceProvider.INSTANCE.getLookupService().writeAsterixInstance(asterixInstance);
-        InstallerUtil.deleteDirectory(InstallerDriver.getManagixHome() + File.separator + InstallerDriver.ASTERIX_DIR
-                + File.separator + asterixInstanceName);
+        AsterixEventServiceUtil.deleteDirectory(InstallerDriver.getManagixHome() + File.separator
+                + InstallerDriver.ASTERIX_DIR + File.separator + asterixInstanceName);
         LOGGER.info(asterixInstance.getDescription(false));
     }
 
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/DeleteCommand.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/DeleteCommand.java
index 00a3de4..3938ea5 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/DeleteCommand.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/DeleteCommand.java
@@ -16,13 +16,14 @@
 
 import org.kohsuke.args4j.Option;
 
+import edu.uci.ics.asterix.event.model.AsterixInstance;
+import edu.uci.ics.asterix.event.model.AsterixInstance.State;
 import edu.uci.ics.asterix.event.schema.pattern.Patterns;
+import edu.uci.ics.asterix.event.service.AsterixEventService;
+import edu.uci.ics.asterix.event.service.AsterixEventServiceUtil;
+import edu.uci.ics.asterix.event.service.ServiceProvider;
+import edu.uci.ics.asterix.event.util.PatternCreator;
 import edu.uci.ics.asterix.installer.driver.InstallerDriver;
-import edu.uci.ics.asterix.installer.driver.InstallerUtil;
-import edu.uci.ics.asterix.installer.events.PatternCreator;
-import edu.uci.ics.asterix.installer.model.AsterixInstance;
-import edu.uci.ics.asterix.installer.model.AsterixInstance.State;
-import edu.uci.ics.asterix.installer.service.ServiceProvider;
 
 public class DeleteCommand extends AbstractCommand {
 
@@ -30,13 +31,13 @@
     protected void execCommand() throws Exception {
         InstallerDriver.initConfig(true);
         String asterixInstanceName = ((DeleteConfig) config).name;
-        AsterixInstance instance = InstallerUtil.validateAsterixInstanceExists(asterixInstanceName, State.INACTIVE);
-        PatternCreator pc = new PatternCreator();
-        Patterns patterns = pc.createDeleteInstancePattern(instance);
-        InstallerUtil.getEventrixClient(instance.getCluster()).submit(patterns);
+        AsterixInstance instance = AsterixEventServiceUtil.validateAsterixInstanceExists(asterixInstanceName,
+                State.INACTIVE);
+        Patterns patterns = PatternCreator.INSTANCE.createDeleteInstancePattern(instance);
+        AsterixEventService.getAsterixEventServiceClient(instance.getCluster()).submit(patterns);
 
-        patterns = pc.createRemoveAsterixWorkingDirPattern(instance);
-        InstallerUtil.getEventrixClient(instance.getCluster()).submit(patterns);
+        patterns = PatternCreator.INSTANCE.createRemoveAsterixWorkingDirPattern(instance);
+        AsterixEventService.getAsterixEventServiceClient(instance.getCluster()).submit(patterns);
         ServiceProvider.INSTANCE.getLookupService().removeAsterixInstance(asterixInstanceName);
         LOGGER.info("Deleted Asterix instance: " + asterixInstanceName);
     }
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/DescribeCommand.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/DescribeCommand.java
index 58851cb..9010061 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/DescribeCommand.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/DescribeCommand.java
@@ -18,14 +18,14 @@
 
 import org.kohsuke.args4j.Option;
 
+import edu.uci.ics.asterix.event.error.VerificationUtil;
+import edu.uci.ics.asterix.event.model.AsterixInstance;
+import edu.uci.ics.asterix.event.model.AsterixInstance.State;
+import edu.uci.ics.asterix.event.model.AsterixRuntimeState;
+import edu.uci.ics.asterix.event.service.AsterixEventServiceUtil;
+import edu.uci.ics.asterix.event.service.ServiceProvider;
 import edu.uci.ics.asterix.installer.driver.InstallerDriver;
-import edu.uci.ics.asterix.installer.driver.InstallerUtil;
 import edu.uci.ics.asterix.installer.error.InstallerException;
-import edu.uci.ics.asterix.installer.error.VerificationUtil;
-import edu.uci.ics.asterix.installer.model.AsterixInstance;
-import edu.uci.ics.asterix.installer.model.AsterixInstance.State;
-import edu.uci.ics.asterix.installer.model.AsterixRuntimeState;
-import edu.uci.ics.asterix.installer.service.ServiceProvider;
 
 public class DescribeCommand extends AbstractCommand {
 
@@ -35,7 +35,7 @@
         String asterixInstanceName = ((DescribeConfig) config).name;
         boolean adminView = ((DescribeConfig) config).admin;
         if (asterixInstanceName != null) {
-            InstallerUtil.validateAsterixInstanceExists(asterixInstanceName, State.INACTIVE, State.ACTIVE,
+            AsterixEventServiceUtil.validateAsterixInstanceExists(asterixInstanceName, State.INACTIVE, State.ACTIVE,
                     State.UNUSABLE);
             AsterixInstance instance = ServiceProvider.INSTANCE.getLookupService().getAsterixInstance(
                     asterixInstanceName);
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/LogCommand.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/LogCommand.java
index 11b4aa7..bf30a48 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/LogCommand.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/LogCommand.java
@@ -21,14 +21,15 @@
 import org.apache.commons.io.FileUtils;
 import org.kohsuke.args4j.Option;
 
-import edu.uci.ics.asterix.event.management.EventrixClient;
+import edu.uci.ics.asterix.event.management.AsterixEventServiceClient;
+import edu.uci.ics.asterix.event.model.AsterixInstance;
+import edu.uci.ics.asterix.event.model.AsterixInstance.State;
 import edu.uci.ics.asterix.event.schema.pattern.Patterns;
+import edu.uci.ics.asterix.event.service.AsterixEventService;
+import edu.uci.ics.asterix.event.service.AsterixEventServiceUtil;
+import edu.uci.ics.asterix.event.util.PatternCreator;
 import edu.uci.ics.asterix.installer.driver.InstallerDriver;
-import edu.uci.ics.asterix.installer.driver.InstallerUtil;
 import edu.uci.ics.asterix.installer.error.InstallerException;
-import edu.uci.ics.asterix.installer.events.PatternCreator;
-import edu.uci.ics.asterix.installer.model.AsterixInstance;
-import edu.uci.ics.asterix.installer.model.AsterixInstance.State;
 
 public class LogCommand extends AbstractCommand {
 
@@ -36,12 +37,15 @@
     protected void execCommand() throws Exception {
         InstallerDriver.initConfig(true);
         String asterixInstanceName = ((LogConfig) config).name;
-        AsterixInstance instance = InstallerUtil.validateAsterixInstanceExists(asterixInstanceName, State.INACTIVE,
-                State.UNUSABLE, State.ACTIVE);
-        PatternCreator pc = new PatternCreator();
-        EventrixClient client = InstallerUtil.getEventrixClient(instance.getCluster());
-        String outputDir = ((LogConfig) config).outputDir == null ? InstallerDriver.getManagixHome() + File.separator + "logdump"
-                : ((LogConfig) config).outputDir;
+        AsterixInstance instance = AsterixEventServiceUtil.validateAsterixInstanceExists(asterixInstanceName,
+                State.INACTIVE, State.UNUSABLE, State.ACTIVE);
+        PatternCreator pc = PatternCreator.INSTANCE;
+
+        AsterixEventServiceClient eventrixClient = AsterixEventService.getAsterixEventServiceClient(
+                instance.getCluster(), true, false);
+
+        String outputDir = ((LogConfig) config).outputDir == null ? InstallerDriver.getManagixHome() + File.separator
+                + "logdump" : ((LogConfig) config).outputDir;
         File f = new File(outputDir);
         String outputDirPath = f.getAbsolutePath();
         if (!f.exists()) {
@@ -50,12 +54,13 @@
                 throw new InstallerException("Unable to create output directory:" + outputDirPath);
             }
         }
-        Patterns transferLogPattern = pc.getGenerateLogPattern(asterixInstanceName, instance.getCluster(), outputDirPath);
-        client.submit(transferLogPattern);
+        Patterns transferLogPattern = pc.getGenerateLogPattern(asterixInstanceName, instance.getCluster(),
+                outputDirPath);
+        eventrixClient.submit(transferLogPattern);
         File outputDirFile = new File(outputDirPath);
         final String destFileName = "log_" + new Date().toString().replace(' ', '_') + ".zip";
         File destFile = new File(outputDirFile, destFileName);
-        InstallerUtil.zipDir(outputDirFile, destFile);
+        AsterixEventServiceUtil.zipDir(outputDirFile, destFile);
 
         String[] filesToDelete = outputDirFile.list(new FilenameFilter() {
             @Override
@@ -65,7 +70,7 @@
 
         });
         for (String fileS : filesToDelete) {
-             f = new File(outputDirFile, fileS);
+            f = new File(outputDirFile, fileS);
             if (f.isDirectory()) {
                 FileUtils.deleteDirectory(f);
             } else {
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/RestoreCommand.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/RestoreCommand.java
index 6627ac2..5b76099 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/RestoreCommand.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/RestoreCommand.java
@@ -18,13 +18,14 @@
 
 import org.kohsuke.args4j.Option;
 
+import edu.uci.ics.asterix.event.model.AsterixInstance;
+import edu.uci.ics.asterix.event.model.AsterixInstance.State;
+import edu.uci.ics.asterix.event.model.BackupInfo;
 import edu.uci.ics.asterix.event.schema.pattern.Patterns;
+import edu.uci.ics.asterix.event.service.AsterixEventService;
+import edu.uci.ics.asterix.event.service.AsterixEventServiceUtil;
+import edu.uci.ics.asterix.event.util.PatternCreator;
 import edu.uci.ics.asterix.installer.driver.InstallerDriver;
-import edu.uci.ics.asterix.installer.driver.InstallerUtil;
-import edu.uci.ics.asterix.installer.events.PatternCreator;
-import edu.uci.ics.asterix.installer.model.AsterixInstance;
-import edu.uci.ics.asterix.installer.model.AsterixInstance.State;
-import edu.uci.ics.asterix.installer.model.BackupInfo;
 
 public class RestoreCommand extends AbstractCommand {
 
@@ -32,7 +33,8 @@
     protected void execCommand() throws Exception {
         InstallerDriver.initConfig(true);
         String asterixInstanceName = ((RestoreConfig) config).name;
-        AsterixInstance instance = InstallerUtil.validateAsterixInstanceExists(asterixInstanceName, State.INACTIVE);
+        AsterixInstance instance = AsterixEventServiceUtil.validateAsterixInstanceExists(asterixInstanceName,
+                State.INACTIVE);
         int backupId = ((RestoreConfig) config).backupId;
         List<BackupInfo> backupInfoList = instance.getBackupInfo();
         if (backupInfoList.size() <= backupId || backupId < 0) {
@@ -40,9 +42,8 @@
         }
 
         BackupInfo backupInfo = backupInfoList.get(backupId);
-        PatternCreator pc = new PatternCreator();
-        Patterns patterns = pc.getRestoreAsterixPattern(instance, backupInfo);
-        InstallerUtil.getEventrixClient(instance.getCluster()).submit(patterns);
+        Patterns patterns = PatternCreator.INSTANCE.getRestoreAsterixPattern(instance, backupInfo);
+        AsterixEventService.getAsterixEventServiceClient(instance.getCluster()).submit(patterns);
         LOGGER.info("Asterix instance: " + asterixInstanceName + " has been restored from backup");
     }
 
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/ShutdownCommand.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/ShutdownCommand.java
index 0c44bd8..228cb1c 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/ShutdownCommand.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/ShutdownCommand.java
@@ -14,9 +14,10 @@
  */
 package edu.uci.ics.asterix.installer.command;
 
+import edu.uci.ics.asterix.event.service.AsterixEventService;
+import edu.uci.ics.asterix.event.service.ILookupService;
+import edu.uci.ics.asterix.event.service.ServiceProvider;
 import edu.uci.ics.asterix.installer.driver.InstallerDriver;
-import edu.uci.ics.asterix.installer.service.ILookupService;
-import edu.uci.ics.asterix.installer.service.ServiceProvider;
 
 public class ShutdownCommand extends AbstractCommand {
 
@@ -24,7 +25,7 @@
     protected void execCommand() throws Exception {
         InstallerDriver.initConfig(false);
         ILookupService lookupService = ServiceProvider.INSTANCE.getLookupService();
-        lookupService.stopService(InstallerDriver.getConfiguration());
+        lookupService.stopService(AsterixEventService.getConfiguration());
     }
 
     @Override
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/StartCommand.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/StartCommand.java
index 3fc7e96..63ec6fc 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/StartCommand.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/StartCommand.java
@@ -18,16 +18,17 @@
 
 import org.kohsuke.args4j.Option;
 
-import edu.uci.ics.asterix.event.management.EventrixClient;
+import edu.uci.ics.asterix.event.error.VerificationUtil;
+import edu.uci.ics.asterix.event.management.AsterixEventServiceClient;
+import edu.uci.ics.asterix.event.model.AsterixInstance;
+import edu.uci.ics.asterix.event.model.AsterixInstance.State;
+import edu.uci.ics.asterix.event.model.AsterixRuntimeState;
 import edu.uci.ics.asterix.event.schema.pattern.Patterns;
+import edu.uci.ics.asterix.event.service.AsterixEventService;
+import edu.uci.ics.asterix.event.service.AsterixEventServiceUtil;
+import edu.uci.ics.asterix.event.service.ServiceProvider;
+import edu.uci.ics.asterix.event.util.PatternCreator;
 import edu.uci.ics.asterix.installer.driver.InstallerDriver;
-import edu.uci.ics.asterix.installer.driver.InstallerUtil;
-import edu.uci.ics.asterix.installer.error.VerificationUtil;
-import edu.uci.ics.asterix.installer.events.PatternCreator;
-import edu.uci.ics.asterix.installer.model.AsterixInstance;
-import edu.uci.ics.asterix.installer.model.AsterixRuntimeState;
-import edu.uci.ics.asterix.installer.model.AsterixInstance.State;
-import edu.uci.ics.asterix.installer.service.ServiceProvider;
 
 public class StartCommand extends AbstractCommand {
 
@@ -35,18 +36,18 @@
     protected void execCommand() throws Exception {
         InstallerDriver.initConfig(true);
         String asterixInstanceName = ((StartConfig) config).name;
-        AsterixInstance instance = InstallerUtil.validateAsterixInstanceExists(asterixInstanceName, State.INACTIVE);
-        InstallerUtil.createAsterixZip(instance);
-        PatternCreator pc = new PatternCreator();
-        EventrixClient client = InstallerUtil.getEventrixClient(instance.getCluster());
-        Patterns asterixBinaryTransferPattern = pc.getAsterixBinaryTransferPattern(asterixInstanceName,
-                instance.getCluster());
+        AsterixInstance instance = AsterixEventServiceUtil.validateAsterixInstanceExists(asterixInstanceName,
+                State.INACTIVE);
+        AsterixEventServiceUtil.createAsterixZip(instance);
+        AsterixEventServiceClient client = AsterixEventService.getAsterixEventServiceClient(instance.getCluster());
+        Patterns asterixBinaryTransferPattern = PatternCreator.INSTANCE.getAsterixBinaryTransferPattern(
+                asterixInstanceName, instance.getCluster());
         client.submit(asterixBinaryTransferPattern);
-        InstallerUtil.createClusterProperties(instance.getCluster(), instance.getAsterixConfiguration());
-        Patterns patterns = pc.getStartAsterixPattern(asterixInstanceName, instance.getCluster());
+        AsterixEventServiceUtil.createClusterProperties(instance.getCluster(), instance.getAsterixConfiguration());
+        Patterns patterns = PatternCreator.INSTANCE.getStartAsterixPattern(asterixInstanceName, instance.getCluster());
         client.submit(patterns);
-        InstallerUtil.deleteDirectory(InstallerDriver.getManagixHome() + File.separator + InstallerDriver.ASTERIX_DIR
-                + File.separator + asterixInstanceName);
+        AsterixEventServiceUtil.deleteDirectory(InstallerDriver.getManagixHome() + File.separator
+                + InstallerDriver.ASTERIX_DIR + File.separator + asterixInstanceName);
         AsterixRuntimeState runtimeState = VerificationUtil.getAsterixRuntimeState(instance);
         VerificationUtil.updateInstanceWithRuntimeDescription(instance, runtimeState, true);
         LOGGER.info(instance.getDescription(false));
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/StopCommand.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/StopCommand.java
index c00fa86..820948c 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/StopCommand.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/StopCommand.java
@@ -20,16 +20,17 @@
 
 import org.kohsuke.args4j.Option;
 
-import edu.uci.ics.asterix.event.management.EventrixClient;
+import edu.uci.ics.asterix.event.management.AsterixEventServiceClient;
+import edu.uci.ics.asterix.event.model.AsterixInstance;
+import edu.uci.ics.asterix.event.model.AsterixInstance.State;
 import edu.uci.ics.asterix.event.schema.cluster.Node;
 import edu.uci.ics.asterix.event.schema.pattern.Pattern;
 import edu.uci.ics.asterix.event.schema.pattern.Patterns;
+import edu.uci.ics.asterix.event.service.AsterixEventService;
+import edu.uci.ics.asterix.event.service.AsterixEventServiceUtil;
+import edu.uci.ics.asterix.event.service.ServiceProvider;
+import edu.uci.ics.asterix.event.util.PatternCreator;
 import edu.uci.ics.asterix.installer.driver.InstallerDriver;
-import edu.uci.ics.asterix.installer.driver.InstallerUtil;
-import edu.uci.ics.asterix.installer.events.PatternCreator;
-import edu.uci.ics.asterix.installer.model.AsterixInstance;
-import edu.uci.ics.asterix.installer.model.AsterixInstance.State;
-import edu.uci.ics.asterix.installer.service.ServiceProvider;
 
 public class StopCommand extends AbstractCommand {
 
@@ -37,18 +38,19 @@
     protected void execCommand() throws Exception {
         InstallerDriver.initConfig(true);
         String asterixInstanceName = ((StopConfig) config).name;
-        AsterixInstance asterixInstance = InstallerUtil.validateAsterixInstanceExists(asterixInstanceName,
+        AsterixInstance asterixInstance = AsterixEventServiceUtil.validateAsterixInstanceExists(asterixInstanceName,
                 State.ACTIVE, State.UNUSABLE);
-        PatternCreator pc = new PatternCreator();
-        EventrixClient client = InstallerUtil.getEventrixClient(asterixInstance.getCluster());
+        AsterixEventServiceClient client = AsterixEventService.getAsterixEventServiceClient(asterixInstance.getCluster());
 
         List<Pattern> ncKillPatterns = new ArrayList<Pattern>();
         for (Node node : asterixInstance.getCluster().getNode()) {
-            ncKillPatterns.add(pc.createNCStopPattern(node.getId(), asterixInstanceName + "_" + node.getId()));
+            ncKillPatterns.add(PatternCreator.INSTANCE.createNCStopPattern(node.getId(), asterixInstanceName + "_"
+                    + node.getId()));
         }
 
         List<Pattern> ccKillPatterns = new ArrayList<Pattern>();
-        ccKillPatterns.add(pc.createCCStopPattern(asterixInstance.getCluster().getMasterNode().getId()));
+        ccKillPatterns.add(PatternCreator.INSTANCE.createCCStopPattern(asterixInstance.getCluster().getMasterNode()
+                .getId()));
 
         try {
             client.submit(new Patterns(ncKillPatterns));
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/ValidateCommand.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/ValidateCommand.java
index 5b63166..317341a 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/ValidateCommand.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/ValidateCommand.java
@@ -29,8 +29,8 @@
 import edu.uci.ics.asterix.event.schema.cluster.Cluster;
 import edu.uci.ics.asterix.event.schema.cluster.MasterNode;
 import edu.uci.ics.asterix.event.schema.cluster.Node;
+import edu.uci.ics.asterix.event.service.AsterixEventServiceUtil;
 import edu.uci.ics.asterix.installer.driver.InstallerDriver;
-import edu.uci.ics.asterix.installer.driver.InstallerUtil;
 import edu.uci.ics.asterix.installer.schema.conf.Configuration;
 import edu.uci.ics.asterix.installer.schema.conf.Zookeeper;
 
@@ -74,7 +74,7 @@
 
     public boolean validateEnvironment() throws Exception {
         boolean valid = true;
-        String managixHome = System.getenv(InstallerDriver.ENV_MANAGIX_HOME);
+        String managixHome = InstallerDriver.getManagixHome();
         if (managixHome == null) {
             valid = false;
             LOGGER.fatal(InstallerDriver.ENV_MANAGIX_HOME + " not set " + ERROR);
@@ -143,7 +143,7 @@
                 + File.separator + "scripts" + File.separator + "validate_ssh.sh";
         List<String> args = ipAddresses;
         args.add(0, username);
-        String output = InstallerUtil.executeLocalScript(script, args);
+        String output = AsterixEventServiceUtil.executeLocalScript(script, args);
         ipAddresses.remove(0);
         for (String line : output.split("\n")) {
             ipAddresses.remove(line);
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerDriver.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerDriver.java
index 521adc6..da0e7ef 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerDriver.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerDriver.java
@@ -15,81 +15,46 @@
 package edu.uci.ics.asterix.installer.driver;
 
 import java.io.File;
-import java.io.FileFilter;
 
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.Unmarshaller;
 
-import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 
+import edu.uci.ics.asterix.event.service.AsterixEventService;
+import edu.uci.ics.asterix.event.service.ILookupService;
+import edu.uci.ics.asterix.event.service.ServiceProvider;
+import edu.uci.ics.asterix.event.util.PatternCreator;
 import edu.uci.ics.asterix.installer.command.CommandHandler;
 import edu.uci.ics.asterix.installer.schema.conf.Configuration;
-import edu.uci.ics.asterix.installer.service.ILookupService;
-import edu.uci.ics.asterix.installer.service.ServiceProvider;
 
 public class InstallerDriver {
 
-    public static final String MANAGIX_INTERNAL_DIR = ".installer";
-    public static final String MANAGIX_EVENT_DIR = MANAGIX_INTERNAL_DIR + File.separator + "eventrix";
-    public static final String MANAGIX_EVENT_SCRIPTS_DIR = MANAGIX_INTERNAL_DIR + File.separator + "eventrix"
-            + File.separator + "scripts";
-    public static final String DEFAULT_ASTERIX_CONFIGURATION_PATH = "conf" + File.separator + File.separator
-            + "asterix-configuration.xml";
-    public static final String ASTERIX_DIR = "asterix";
-    public static final String EVENTS_DIR = "events";
-
     private static final Logger LOGGER = Logger.getLogger(InstallerDriver.class.getName());
+
+    public static final String MANAGIX_INTERNAL_DIR = ".installer";
     public static final String ENV_MANAGIX_HOME = "MANAGIX_HOME";
     public static final String MANAGIX_CONF_XML = "conf" + File.separator + "managix-conf.xml";
+    public static final String ASTERIX_DIR = "asterix";
 
-    private static Configuration conf;
     private static String managixHome;
-    private static String asterixZip;
-
-    public static String getAsterixZip() {
-        return asterixZip;
-    }
-
-    public static Configuration getConfiguration() {
-        return conf;
-    }
 
     public static void initConfig(boolean ensureLookupServiceIsRunning) throws Exception {
         File configFile = new File(managixHome + File.separator + MANAGIX_CONF_XML);
         JAXBContext configCtx = JAXBContext.newInstance(Configuration.class);
         Unmarshaller unmarshaller = configCtx.createUnmarshaller();
-        conf = (Configuration) unmarshaller.unmarshal(configFile);
-        asterixZip = initBinary("asterix-server");
+        Configuration conf = (Configuration) unmarshaller.unmarshal(configFile);
+        String asterixDir = managixHome + File.separator + ASTERIX_DIR;
+        String eventHome = managixHome + File.separator + MANAGIX_INTERNAL_DIR;
+        AsterixEventService.initialize(conf, asterixDir, eventHome);
 
+        
         ILookupService lookupService = ServiceProvider.INSTANCE.getLookupService();
         if (ensureLookupServiceIsRunning && !lookupService.isRunning(conf)) {
             lookupService.startService(conf);
         }
     }
 
-    private static String initBinary(final String fileNamePattern) {
-        String asterixDir = InstallerDriver.getAsterixDir();
-        File file = new File(asterixDir);
-        File[] zipFiles = file.listFiles(new FileFilter() {
-            public boolean accept(File arg0) {
-                return arg0.getAbsolutePath().contains(fileNamePattern) && arg0.isFile();
-            }
-        });
-        if (zipFiles.length == 0) {
-            String msg = " Binary not found at " + asterixDir;
-            LOGGER.log(Level.FATAL, msg);
-            throw new IllegalStateException(msg);
-        }
-        if (zipFiles.length > 1) {
-            String msg = " Multiple binaries found at " + asterixDir;
-            LOGGER.log(Level.FATAL, msg);
-            throw new IllegalStateException(msg);
-        }
-
-        return zipFiles[0].getAbsolutePath();
-    }
-
     public static String getManagixHome() {
         return managixHome;
     }
@@ -98,10 +63,6 @@
         InstallerDriver.managixHome = managixHome;
     }
 
-    public static String getAsterixDir() {
-        return managixHome + File.separator + ASTERIX_DIR;
-    }
-
     public static void main(String args[]) {
         try {
             if (args.length != 0) {
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerUtil.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerUtil.java
index e62cc59..2569363 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerUtil.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerUtil.java
@@ -14,58 +14,23 @@
  */
 package edu.uci.ics.asterix.installer.driver;
 
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
 import java.io.File;
-import java.io.FileFilter;
-import java.io.FileInputStream;
 import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FileWriter;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
-import java.util.Enumeration;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Random;
-import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
-import java.util.jar.JarOutputStream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-import java.util.zip.ZipOutputStream;
 
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.JAXBException;
-import javax.xml.bind.Marshaller;
 import javax.xml.bind.Unmarshaller;
 
-import org.apache.commons.io.IOUtils;
-
 import edu.uci.ics.asterix.common.configuration.AsterixConfiguration;
-import edu.uci.ics.asterix.common.configuration.Coredump;
-import edu.uci.ics.asterix.common.configuration.Store;
-import edu.uci.ics.asterix.common.configuration.TransactionLogDir;
-import edu.uci.ics.asterix.event.driver.EventDriver;
-import edu.uci.ics.asterix.event.management.EventUtil;
-import edu.uci.ics.asterix.event.management.EventrixClient;
 import edu.uci.ics.asterix.event.schema.cluster.Cluster;
-import edu.uci.ics.asterix.event.schema.cluster.Env;
 import edu.uci.ics.asterix.event.schema.cluster.Node;
-import edu.uci.ics.asterix.event.schema.cluster.Property;
-import edu.uci.ics.asterix.installer.error.InstallerException;
-import edu.uci.ics.asterix.installer.error.OutputHandler;
-import edu.uci.ics.asterix.installer.model.AsterixInstance;
-import edu.uci.ics.asterix.installer.model.AsterixInstance.State;
-import edu.uci.ics.asterix.installer.service.ServiceProvider;
 
 public class InstallerUtil {
 
+    private static final String DEFAULT_ASTERIX_CONFIGURATION_PATH = "conf" + File.separator
+            + "asterix-configuration.xml";
+
     public static final String TXN_LOG_DIR = "txnLogs";
     public static final String TXN_LOG_DIR_KEY_SUFFIX = "txnLogDir";
     public static final String ASTERIX_CONFIGURATION_FILE = "asterix-configuration.xml";
@@ -75,133 +40,6 @@
     public static final int HTTP_PORT_DEFAULT = 8888;
     public static final int WEB_INTERFACE_PORT_DEFAULT = 19001;
 
-    public static AsterixInstance createAsterixInstance(String asterixInstanceName, Cluster cluster,
-            AsterixConfiguration asterixConfiguration) throws FileNotFoundException, IOException {
-        Node metadataNode = getMetadataNode(cluster);
-        String asterixZipName = InstallerDriver.getAsterixZip().substring(
-                InstallerDriver.getAsterixZip().lastIndexOf(File.separator) + 1);
-        String asterixVersion = asterixZipName.substring("asterix-server-".length(),
-                asterixZipName.indexOf("-binary-assembly"));
-        AsterixInstance instance = new AsterixInstance(asterixInstanceName, cluster, asterixConfiguration,
-                metadataNode.getId(), asterixVersion);
-        return instance;
-    }
-
-    public static void createAsterixZip(AsterixInstance asterixInstance) throws IOException, InterruptedException,
-            JAXBException, InstallerException {
-
-        String modifiedZipPath = injectAsterixPropertyFile(InstallerDriver.getAsterixZip(), asterixInstance);
-        injectAsterixLogPropertyFile(modifiedZipPath, asterixInstance);
-    }
-
-    public static void createClusterProperties(Cluster cluster, AsterixConfiguration asterixConfiguration) {
-        List<Property> clusterProperties = null;
-        if (cluster.getEnv() != null && cluster.getEnv().getProperty() != null) {
-            clusterProperties = cluster.getEnv().getProperty();
-            clusterProperties.clear();
-        } else {
-            clusterProperties = new ArrayList<Property>();
-        }
-        for (edu.uci.ics.asterix.common.configuration.Property property : asterixConfiguration.getProperty()) {
-            if (property.getName().equalsIgnoreCase(EventUtil.CC_JAVA_OPTS)) {
-                clusterProperties.add(new Property(EventUtil.CC_JAVA_OPTS, property.getValue()));
-            } else if (property.getName().equalsIgnoreCase(EventUtil.NC_JAVA_OPTS)) {
-                clusterProperties.add(new Property(EventUtil.NC_JAVA_OPTS, property.getValue()));
-            }
-        }
-        clusterProperties.add(new Property("ASTERIX_HOME", cluster.getWorkingDir().getDir() + File.separator
-                + "asterix"));
-        clusterProperties.add(new Property("LOG_DIR", cluster.getLogDir()));
-        clusterProperties.add(new Property("JAVA_HOME", cluster.getJavaHome()));
-        clusterProperties.add(new Property("WORKING_DIR", cluster.getWorkingDir().getDir()));
-        clusterProperties.add(new Property("CLIENT_NET_IP", cluster.getMasterNode().getClientIp()));
-        clusterProperties.add(new Property("CLUSTER_NET_IP", cluster.getMasterNode().getClusterIp()));
-
-        int clusterNetPort = cluster.getMasterNode().getClusterPort() != null ? cluster.getMasterNode()
-                .getClusterPort().intValue() : CLUSTER_NET_PORT_DEFAULT;
-        int clientNetPort = cluster.getMasterNode().getClientPort() != null ? cluster.getMasterNode().getClientPort()
-                .intValue() : CLIENT_NET_PORT_DEFAULT;
-        int httpPort = cluster.getMasterNode().getHttpPort() != null ? cluster.getMasterNode().getHttpPort().intValue()
-                : HTTP_PORT_DEFAULT;
-
-        clusterProperties.add(new Property("CLIENT_NET_PORT", "" + clientNetPort));
-        clusterProperties.add(new Property("CLUSTER_NET_PORT", "" + clusterNetPort));
-        clusterProperties.add(new Property("HTTP_PORT", "" + httpPort));
-
-        cluster.setEnv(new Env(clusterProperties));
-    }
-
-    private static String injectAsterixPropertyFile(String origZipFile, AsterixInstance asterixInstance)
-            throws IOException, JAXBException {
-        writeAsterixConfigurationFile(asterixInstance);
-        String asterixInstanceDir = InstallerDriver.getAsterixDir() + File.separator + asterixInstance.getName();
-        unzip(origZipFile, asterixInstanceDir);
-        File sourceJar = new File(asterixInstanceDir + File.separator + "lib" + File.separator + "asterix-app-"
-                + asterixInstance.getAsterixVersion() + ".jar");
-        File replacementFile = new File(asterixInstanceDir + File.separator + ASTERIX_CONFIGURATION_FILE);
-        replaceInJar(sourceJar, ASTERIX_CONFIGURATION_FILE, replacementFile);
-        new File(asterixInstanceDir + File.separator + ASTERIX_CONFIGURATION_FILE).delete();
-        String asterixZipName = InstallerDriver.getAsterixZip().substring(
-                InstallerDriver.getAsterixZip().lastIndexOf(File.separator) + 1);
-        zipDir(new File(asterixInstanceDir), new File(asterixInstanceDir + File.separator + asterixZipName));
-        return asterixInstanceDir + File.separator + asterixZipName;
-    }
-
-    private static String injectAsterixLogPropertyFile(String origZipFile, AsterixInstance asterixInstance)
-            throws IOException, InstallerException {
-        String asterixInstanceDir = InstallerDriver.getAsterixDir() + File.separator + asterixInstance.getName();
-        unzip(origZipFile, asterixInstanceDir);
-        File sourceJar1 = new File(asterixInstanceDir + File.separator + "lib" + File.separator + "asterix-app-"
-                + asterixInstance.getAsterixVersion() + ".jar");
-        Properties txnLogProperties = new Properties();
-        URLClassLoader urlClassLoader = new URLClassLoader(new URL[] { sourceJar1.toURI().toURL() });
-        InputStream in = urlClassLoader.getResourceAsStream(TXN_LOG_CONFIGURATION_FILE);
-        if (in != null) {
-            txnLogProperties.load(in);
-        }
-
-        writeAsterixLogConfigurationFile(asterixInstance, txnLogProperties);
-
-        File sourceJar2 = new File(asterixInstanceDir + File.separator + "lib" + File.separator + "asterix-app-"
-                + asterixInstance.getAsterixVersion() + ".jar");
-        File replacementFile = new File(asterixInstanceDir + File.separator + "log.properties");
-        replaceInJar(sourceJar2, TXN_LOG_CONFIGURATION_FILE, replacementFile);
-
-        new File(asterixInstanceDir + File.separator + "log.properties").delete();
-        String asterixZipName = InstallerDriver.getAsterixZip().substring(
-                InstallerDriver.getAsterixZip().lastIndexOf(File.separator) + 1);
-        zipDir(new File(asterixInstanceDir), new File(asterixInstanceDir + File.separator + asterixZipName));
-        return asterixInstanceDir + File.separator + asterixZipName;
-    }
-
-    public static void addLibraryToAsterixZip(AsterixInstance asterixInstance, String dataverseName,
-            String libraryName, String libraryPath) throws IOException {
-        File instanceDir = new File(InstallerDriver.getAsterixDir() + File.separator + asterixInstance.getName());
-        if (!instanceDir.exists()) {
-            instanceDir.mkdirs();
-        }
-        String asterixZipName = InstallerDriver.getAsterixZip().substring(
-                InstallerDriver.getAsterixZip().lastIndexOf(File.separator) + 1);
-
-        String sourceZip = instanceDir.getAbsolutePath() + File.separator + asterixZipName;
-        unzip(sourceZip, instanceDir.getAbsolutePath());
-        File libraryPathInZip = new File(instanceDir.getAbsolutePath() + File.separator + "external" + File.separator
-                + "library" + dataverseName + File.separator + "to-add" + File.separator + libraryName);
-        libraryPathInZip.mkdirs();
-        Runtime.getRuntime().exec("cp" + " " + libraryPath + " " + libraryPathInZip.getAbsolutePath());
-        Runtime.getRuntime().exec("rm " + sourceZip);
-        String destZip = InstallerDriver.getAsterixDir() + File.separator + asterixInstance.getName() + File.separator
-                + asterixZipName;
-        zipDir(instanceDir, new File(destZip));
-        Runtime.getRuntime().exec("mv" + " " + destZip + " " + sourceZip);
-    }
-
-    private static Node getMetadataNode(Cluster cluster) {
-        Random random = new Random();
-        int nNodes = cluster.getNode().size();
-        return cluster.getNode().get(random.nextInt(nNodes));
-    }
-
     public static String getNodeDirectories(String asterixInstanceName, Node node, Cluster cluster) {
         String storeDataSubDir = asterixInstanceName + File.separator + "data" + File.separator;
         String[] storeDirs = null;
@@ -224,82 +62,10 @@
         return nodeDataStore.toString();
     }
 
-    private static void writeAsterixConfigurationFile(AsterixInstance asterixInstance) throws IOException,
-            JAXBException {
-        String asterixInstanceName = asterixInstance.getName();
-        Cluster cluster = asterixInstance.getCluster();
-        String metadataNodeId = asterixInstance.getMetadataNodeId();
-
-        AsterixConfiguration configuration = asterixInstance.getAsterixConfiguration();
-        configuration.setMetadataNode(asterixInstanceName + "_" + metadataNodeId);
-
-        String storeDir = null;
-        List<Store> stores = new ArrayList<Store>();
-        for (Node node : cluster.getNode()) {
-            storeDir = node.getStore() == null ? cluster.getStore() : node.getStore();
-            stores.add(new Store(asterixInstanceName + "_" + node.getId(), storeDir));
-        }
-        configuration.setStore(stores);
-
-        List<Coredump> coredump = new ArrayList<Coredump>();
-        String coredumpDir = null;
-        List<TransactionLogDir> txnLogDirs = new ArrayList<TransactionLogDir>();
-        String txnLogDir = null;
-        for (Node node : cluster.getNode()) {
-            coredumpDir = node.getLogDir() == null ? cluster.getLogDir() : node.getLogDir();
-            coredump.add(new Coredump(asterixInstanceName + "_" + node.getId(), coredumpDir  + File.separator + asterixInstanceName +  "_"  + node.getId()));
-
-            txnLogDir = node.getTxnLogDir() == null ? cluster.getTxnLogDir() : node.getTxnLogDir();
-            txnLogDirs.add(new TransactionLogDir(asterixInstanceName + "_" + node.getId(), txnLogDir));
-        }
-        configuration.setCoredump(coredump);
-        configuration.setTransactionLogDir(txnLogDirs);
-
-        File asterixConfDir = new File(InstallerDriver.getAsterixDir() + File.separator + asterixInstanceName);
-        asterixConfDir.mkdirs();
-
-        JAXBContext ctx = JAXBContext.newInstance(AsterixConfiguration.class);
-        Marshaller marshaller = ctx.createMarshaller();
-        marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
-        marshaller.marshal(configuration, new FileOutputStream(asterixConfDir + File.separator
-                + ASTERIX_CONFIGURATION_FILE));
-    }
-
-    private static void writeAsterixLogConfigurationFile(AsterixInstance asterixInstance, Properties logProperties)
-            throws IOException, InstallerException {
-        String asterixInstanceName = asterixInstance.getName();
-        Cluster cluster = asterixInstance.getCluster();
-        StringBuffer conf = new StringBuffer();
-        for (Map.Entry<Object, Object> p : logProperties.entrySet()) {
-            conf.append(p.getKey() + "=" + p.getValue() + "\n");
-        }
-
-        for (Node node : cluster.getNode()) {
-            String txnLogDir = node.getTxnLogDir() == null ? cluster.getTxnLogDir() : node.getTxnLogDir();
-            if (txnLogDir == null) {
-                throw new InstallerException("Transaction log directory (txn_log_dir) not configured for node: "
-                        + node.getId());
-            }
-            conf.append(asterixInstanceName + "_" + node.getId() + "." + TXN_LOG_DIR_KEY_SUFFIX + "=" + txnLogDir
-                    + "\n");
-        }
-        List<edu.uci.ics.asterix.common.configuration.Property> properties = asterixInstance.getAsterixConfiguration()
-                .getProperty();
-        for (edu.uci.ics.asterix.common.configuration.Property p : properties) {
-            if (p.getName().trim().toLowerCase().contains("log")) {
-                conf.append(p.getValue() + "=" + p.getValue());
-            }
-        }
-        dumpToFile(InstallerDriver.getAsterixDir() + File.separator + asterixInstanceName + File.separator
-                + "log.properties", conf.toString());
-
-    }
-
     public static AsterixConfiguration getAsterixConfiguration(String asterixConf) throws FileNotFoundException,
             IOException, JAXBException {
         if (asterixConf == null) {
-            asterixConf = InstallerDriver.getManagixHome() + File.separator
-                    + InstallerDriver.DEFAULT_ASTERIX_CONFIGURATION_PATH;
+            asterixConf = InstallerDriver.getManagixHome() + File.separator + DEFAULT_ASTERIX_CONFIGURATION_PATH;
         }
         File file = new File(asterixConf);
         JAXBContext ctx = JAXBContext.newInstance(AsterixConfiguration.class);
@@ -308,197 +74,4 @@
         return asterixConfiguration;
     }
 
-    public static void unzip(String sourceFile, String destDir) throws IOException {
-        BufferedOutputStream dest = null;
-        FileInputStream fis = new FileInputStream(sourceFile);
-        ZipInputStream zis = new ZipInputStream(new BufferedInputStream(fis));
-        ZipEntry entry = null;
-
-        int BUFFER_SIZE = 4096;
-        while ((entry = zis.getNextEntry()) != null) {
-            String dst = destDir + File.separator + entry.getName();
-            if (entry.isDirectory()) {
-                createDir(destDir, entry);
-                continue;
-            }
-            int count;
-            byte data[] = new byte[BUFFER_SIZE];
-
-            // write the file to the disk
-            FileOutputStream fos = new FileOutputStream(dst);
-            dest = new BufferedOutputStream(fos, BUFFER_SIZE);
-            while ((count = zis.read(data, 0, BUFFER_SIZE)) != -1) {
-                dest.write(data, 0, count);
-            }
-            // close the output streams
-            dest.flush();
-            dest.close();
-        }
-
-        zis.close();
-    }
-
-    public static void zipDir(File sourceDir, File destFile) throws IOException {
-        FileOutputStream fos = new FileOutputStream(destFile);
-        ZipOutputStream zos = new ZipOutputStream(fos);
-        zipDir(sourceDir, destFile, zos);
-        zos.close();
-    }
-
-    private static void zipDir(File sourceDir, final File destFile, ZipOutputStream zos) throws IOException {
-        File[] dirList = sourceDir.listFiles(new FileFilter() {
-            public boolean accept(File f) {
-                return !f.getName().endsWith(destFile.getName());
-            }
-        });
-        for (int i = 0; i < dirList.length; i++) {
-            File f = dirList[i];
-            if (f.isDirectory()) {
-                zipDir(f, destFile, zos);
-            } else {
-                int bytesIn = 0;
-                byte[] readBuffer = new byte[2156];
-                FileInputStream fis = new FileInputStream(f);
-                ZipEntry entry = new ZipEntry(sourceDir.getName() + File.separator + f.getName());
-                zos.putNextEntry(entry);
-                while ((bytesIn = fis.read(readBuffer)) != -1) {
-                    zos.write(readBuffer, 0, bytesIn);
-                }
-                fis.close();
-            }
-        }
-    }
-
-    private static void replaceInJar(File sourceJar, String origFile, File replacementFile) throws IOException {
-        File destJar = new File(sourceJar.getAbsolutePath() + ".modified");
-        InputStream jarIs = null;
-        FileInputStream fis = new FileInputStream(replacementFile);
-        JarFile sourceJarFile = new JarFile(sourceJar);
-        Enumeration<JarEntry> entries = sourceJarFile.entries();
-        JarOutputStream jos = new JarOutputStream(new FileOutputStream(destJar));
-        byte[] buffer = new byte[2048];
-        int read;
-        while (entries.hasMoreElements()) {
-            JarEntry entry = (JarEntry) entries.nextElement();
-            String name = entry.getName();
-            if (name.equals(origFile)) {
-                continue;
-            }
-            jarIs = sourceJarFile.getInputStream(entry);
-            jos.putNextEntry(entry);
-            while ((read = jarIs.read(buffer)) != -1) {
-                jos.write(buffer, 0, read);
-            }
-        }
-        JarEntry entry = new JarEntry(origFile);
-        jos.putNextEntry(entry);
-        while ((read = fis.read(buffer)) != -1) {
-            jos.write(buffer, 0, read);
-        }
-        fis.close();
-        jos.close();
-        jarIs.close();
-        sourceJar.delete();
-        destJar.renameTo(sourceJar);
-        sourceJar.setExecutable(true);
-    }
-
-    public static void dumpToFile(String dest, String content) throws IOException {
-        FileWriter writer = new FileWriter(dest);
-        writer.write(content);
-        writer.close();
-    }
-
-    private static void createDir(String destDirectory, ZipEntry entry) {
-        String name = entry.getName();
-        int index = name.lastIndexOf(File.separator);
-        String dirSequence = name.substring(0, index);
-        File newDirs = new File(destDirectory + File.separator + dirSequence);
-        newDirs.mkdirs();
-    }
-
-    public static AsterixInstance validateAsterixInstanceExists(String name, State... permissibleStates)
-            throws Exception {
-        AsterixInstance instance = ServiceProvider.INSTANCE.getLookupService().getAsterixInstance(name);
-        if (instance == null) {
-            throw new InstallerException("Asterix instance by name " + name + " does not exist.");
-        }
-        boolean valid = false;
-        for (State state : permissibleStates) {
-            if (state.equals(instance.getState())) {
-                valid = true;
-                break;
-            }
-        }
-        if (!valid) {
-            throw new InstallerException("Asterix instance by the name " + name + " is in " + instance.getState()
-                    + " state ");
-        }
-        return instance;
-    }
-
-    public static void validateAsterixInstanceNotExists(String name) throws Exception {
-        AsterixInstance instance = ServiceProvider.INSTANCE.getLookupService().getAsterixInstance(name);
-        if (instance != null) {
-            throw new InstallerException("Asterix instance by name " + name + " already exists.");
-        }
-    }
-
-    public static void evaluateConflictWithOtherInstances(AsterixInstance instance) throws Exception {
-        List<AsterixInstance> existingInstances = ServiceProvider.INSTANCE.getLookupService().getAsterixInstances();
-        List<String> usedIps = new ArrayList<String>();
-        String masterIp = instance.getCluster().getMasterNode().getClusterIp();
-        for (Node node : instance.getCluster().getNode()) {
-            usedIps.add(node.getClusterIp());
-        }
-        usedIps.add(instance.getCluster().getMasterNode().getClusterIp());
-        boolean conflictFound = false;
-        AsterixInstance conflictingInstance = null;
-        for (AsterixInstance existing : existingInstances) {
-            conflictFound = existing.getCluster().getMasterNode().getClusterIp().equals(masterIp);
-            if (conflictFound) {
-                conflictingInstance = existing;
-                break;
-            }
-            for (Node n : existing.getCluster().getNode()) {
-                if (usedIps.contains(n.getClusterIp())) {
-                    conflictFound = true;
-                    conflictingInstance = existing;
-                    break;
-                }
-            }
-        }
-        if (conflictFound) {
-            throw new Exception("Cluster definition conflicts with an existing instance of Asterix: "
-                    + conflictingInstance.getName());
-        }
-    }
-
-    public static void deleteDirectory(String path) throws IOException {
-        Runtime.getRuntime().exec("rm -rf " + path);
-    }
-
-    public static String executeLocalScript(String path, List<String> args) throws Exception {
-        List<String> pargs = new ArrayList<String>();
-        pargs.add("/bin/bash");
-        pargs.add(path);
-        if (args != null) {
-            pargs.addAll(args);
-        }
-        ProcessBuilder pb = new ProcessBuilder(pargs);
-        pb.environment().putAll(EventDriver.getEnvironment());
-        pb.environment().put("IP_LOCATION", EventDriver.CLIENT_NODE.getClusterIp());
-        Process p = pb.start();
-        BufferedInputStream bis = new BufferedInputStream(p.getInputStream());
-        StringWriter writer = new StringWriter();
-        IOUtils.copy(bis, writer, "UTF-8");
-        return writer.toString();
-    }
-
-    public static EventrixClient getEventrixClient(Cluster cluster) throws Exception {
-        return new EventrixClient(
-                InstallerDriver.getManagixHome() + File.separator + InstallerDriver.MANAGIX_EVENT_DIR, cluster, false,
-                OutputHandler.INSTANCE);
-    }
-
 }
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/error/OutputHandler.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/error/OutputHandler.java
index 313fa50..0a86196 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/error/OutputHandler.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/error/OutputHandler.java
@@ -20,8 +20,8 @@
 
 import edu.uci.ics.asterix.event.management.IOutputHandler;
 import edu.uci.ics.asterix.event.management.OutputAnalysis;
+import edu.uci.ics.asterix.event.model.EventList.EventType;
 import edu.uci.ics.asterix.event.schema.pattern.Event;
-import edu.uci.ics.asterix.installer.model.EventList.EventType;
 
 public class OutputHandler implements IOutputHandler {
 
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/error/VerificationUtil.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/error/VerificationUtil.java
deleted file mode 100644
index 5be3f89..0000000
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/error/VerificationUtil.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.installer.error;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.asterix.event.schema.cluster.Cluster;
-import edu.uci.ics.asterix.event.schema.cluster.Node;
-import edu.uci.ics.asterix.installer.driver.InstallerDriver;
-import edu.uci.ics.asterix.installer.driver.InstallerUtil;
-import edu.uci.ics.asterix.installer.model.AsterixInstance;
-import edu.uci.ics.asterix.installer.model.AsterixInstance.State;
-import edu.uci.ics.asterix.installer.model.AsterixRuntimeState;
-import edu.uci.ics.asterix.installer.model.ProcessInfo;
-
-public class VerificationUtil {
-
-	private static final String VERIFY_SCRIPT_PATH = InstallerDriver
-			.getManagixHome()
-			+ File.separator
-			+ InstallerDriver.MANAGIX_INTERNAL_DIR
-			+ File.separator
-			+ "scripts"
-			+ File.separator + "verify.sh";
-
-	public static AsterixRuntimeState getAsterixRuntimeState(
-			AsterixInstance instance) throws Exception {
-
-		Cluster cluster = instance.getCluster();
-		List<String> args = new ArrayList<String>();
-		args.add(instance.getName());
-		args.add(instance.getCluster().getMasterNode().getClusterIp());
-		for (Node node : cluster.getNode()) {
-			args.add(node.getClusterIp());
-			args.add(instance.getName() + "_" + node.getId());
-		}
-		Thread.sleep(2000);
-		String output = InstallerUtil.executeLocalScript(VERIFY_SCRIPT_PATH,
-				args);
-		boolean ccRunning = true;
-		List<String> failedNCs = new ArrayList<String>();
-		String[] infoFields;
-		ProcessInfo pInfo;
-		List<ProcessInfo> processes = new ArrayList<ProcessInfo>();
-
-		for (String line : output.split("\n")) {
-			String nodeid = null;
-			infoFields = line.split(":");
-			try {
-				int pid = Integer.parseInt(infoFields[3]);
-				if (infoFields[0].equals("NC")) {
-					nodeid = infoFields[2].split("_")[1];
-				} else {
-					nodeid = instance.getCluster().getMasterNode().getId();
-				}
-				pInfo = new ProcessInfo(infoFields[0], infoFields[1], nodeid,
-						pid);
-				processes.add(pInfo);
-			} catch (Exception e) {
-				if (infoFields[0].equalsIgnoreCase("CC")) {
-					ccRunning = false;
-				} else {
-					failedNCs.add(infoFields[1]);
-				}
-			}
-		}
-		return new AsterixRuntimeState(processes, failedNCs, ccRunning);
-	}
-
-	public static void updateInstanceWithRuntimeDescription(
-			AsterixInstance instance, AsterixRuntimeState state,
-			boolean expectedRunning) {
-		StringBuffer summary = new StringBuffer();
-		if (expectedRunning) {
-			if (!state.isCcRunning()) {
-				summary.append("Cluster Controller not running at "
-						+ instance.getCluster().getMasterNode().getId() + "\n");
-				instance.setState(State.UNUSABLE);
-			}
-			if (state.getFailedNCs() != null && !state.getFailedNCs().isEmpty()) {
-				summary.append("Node Controller not running at the following nodes"
-						+ "\n");
-				for (String failedNC : state.getFailedNCs()) {
-					summary.append(failedNC + "\n");
-				}
-				instance.setState(State.UNUSABLE);
-			}
-			if (!(instance.getState().equals(State.UNUSABLE))) {
-				instance.setState(State.ACTIVE);
-			}
-		} else {
-			if (state.getProcesses() != null && state.getProcesses().size() > 0) {
-				summary.append("Following process still running " + "\n");
-				for (ProcessInfo pInfo : state.getProcesses()) {
-					summary.append(pInfo + "\n");
-				}
-				instance.setState(State.UNUSABLE);
-			} else {
-				instance.setState(State.INACTIVE);
-			}
-		}
-		state.setSummary(summary.toString());
-		instance.setAsterixRuntimeStates(state);
-	}
-
-	public static void verifyBackupRestoreConfiguration(String hdfsUrl,
-			String hadoopVersion, String hdfsBackupDir) throws Exception {
-		StringBuffer errorCheck = new StringBuffer();
-		if (hdfsUrl == null || hdfsUrl.length() == 0) {
-			errorCheck.append("\n HDFS Url not configured");
-		}
-		if (hadoopVersion == null || hadoopVersion.length() == 0) {
-			errorCheck.append("\n HDFS version not configured");
-		}
-		if (hdfsBackupDir == null || hdfsBackupDir.length() == 0) {
-			errorCheck.append("\n HDFS backup directory not configured");
-		}
-		if (errorCheck.length() > 0) {
-			throw new Exception("Incomplete hdfs configuration in "
-					+ InstallerDriver.getManagixHome() + File.separator
-					+ InstallerDriver.MANAGIX_CONF_XML + errorCheck);
-		}
-	}
-}
diff --git a/asterix-installer/src/main/resources/hadoop-0.20.2/bin/hadoop b/asterix-installer/src/main/resources/hadoop-0.20.2/bin/hadoop
old mode 100755
new mode 100644
diff --git a/asterix-installer/src/main/resources/hadoop-0.20.2/bin/hadoop-config.sh b/asterix-installer/src/main/resources/hadoop-0.20.2/bin/hadoop-config.sh
old mode 100755
new mode 100644
diff --git a/asterix-installer/src/main/resources/scripts/managix b/asterix-installer/src/main/resources/scripts/managix
old mode 100755
new mode 100644
diff --git a/asterix-installer/src/main/resources/scripts/validate_ssh.sh b/asterix-installer/src/main/resources/scripts/validate_ssh.sh
old mode 100755
new mode 100644
diff --git a/asterix-installer/src/main/resources/scripts/verify.sh b/asterix-installer/src/main/resources/scripts/verify.sh
old mode 100755
new mode 100644
diff --git a/asterix-installer/src/main/resources/zookeeper/start_zk.sh b/asterix-installer/src/main/resources/zookeeper/start_zk.sh
old mode 100755
new mode 100644
diff --git a/asterix-installer/src/main/resources/zookeeper/stop_zk b/asterix-installer/src/main/resources/zookeeper/stop_zk
old mode 100755
new mode 100644
diff --git a/asterix-installer/src/main/resources/zookeeper/zk.init b/asterix-installer/src/main/resources/zookeeper/zk.init
old mode 100755
new mode 100644
diff --git a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixInstallerIntegrationUtil.java b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixInstallerIntegrationUtil.java
index c7beb28..9926f01 100644
--- a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixInstallerIntegrationUtil.java
+++ b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixInstallerIntegrationUtil.java
@@ -28,15 +28,14 @@
 import javax.xml.bind.Marshaller;
 import javax.xml.bind.Unmarshaller;
 
+import edu.uci.ics.asterix.event.error.VerificationUtil;
+import edu.uci.ics.asterix.event.model.AsterixInstance;
+import edu.uci.ics.asterix.event.model.AsterixInstance.State;
+import edu.uci.ics.asterix.event.model.AsterixRuntimeState;
+import edu.uci.ics.asterix.event.service.ServiceProvider;
 import edu.uci.ics.asterix.installer.command.CommandHandler;
-import edu.uci.ics.asterix.installer.command.ShutdownCommand;
 import edu.uci.ics.asterix.installer.driver.InstallerDriver;
-import edu.uci.ics.asterix.installer.error.VerificationUtil;
-import edu.uci.ics.asterix.installer.model.AsterixInstance;
-import edu.uci.ics.asterix.installer.model.AsterixInstance.State;
-import edu.uci.ics.asterix.installer.model.AsterixRuntimeState;
 import edu.uci.ics.asterix.installer.schema.conf.Configuration;
-import edu.uci.ics.asterix.installer.service.ServiceProvider;
 import edu.uci.ics.hyracks.api.client.HyracksConnection;
 import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 
diff --git a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixLifecycleIT.java b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixLifecycleIT.java
index c8f6cee..421eb42 100644
--- a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixLifecycleIT.java
+++ b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixLifecycleIT.java
@@ -25,12 +25,12 @@
 import org.junit.Test;
 import org.junit.runners.Parameterized.Parameters;
 
+import edu.uci.ics.asterix.event.error.VerificationUtil;
+import edu.uci.ics.asterix.event.model.AsterixInstance;
+import edu.uci.ics.asterix.event.model.AsterixInstance.State;
+import edu.uci.ics.asterix.event.model.AsterixRuntimeState;
+import edu.uci.ics.asterix.event.service.ServiceProvider;
 import edu.uci.ics.asterix.installer.command.CommandHandler;
-import edu.uci.ics.asterix.installer.error.VerificationUtil;
-import edu.uci.ics.asterix.installer.model.AsterixInstance;
-import edu.uci.ics.asterix.installer.model.AsterixInstance.State;
-import edu.uci.ics.asterix.installer.model.AsterixRuntimeState;
-import edu.uci.ics.asterix.installer.service.ServiceProvider;
 import edu.uci.ics.asterix.test.aql.TestsUtils;
 import edu.uci.ics.asterix.testframework.context.TestCaseContext;
 
diff --git a/asterix-metadata/pom.xml b/asterix-metadata/pom.xml
index 6b1e8cf..febe440 100644
--- a/asterix-metadata/pom.xml
+++ b/asterix-metadata/pom.xml
@@ -1,18 +1,14 @@
-<!--
- ! Copyright 2009-2013 by The Regents of the University of California
- ! Licensed under the Apache License, Version 2.0 (the "License");
- ! you may not use this file except in compliance with the License.
- ! you may obtain a copy of the License from
- ! 
- !     http://www.apache.org/licenses/LICENSE-2.0
- ! 
- ! Unless required by applicable law or agreed to in writing, software
- ! distributed under the License is distributed on an "AS IS" BASIS,
- ! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ! See the License for the specific language governing permissions and
- ! limitations under the License.
- !-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<!-- ! Copyright 2009-2013 by The Regents of the University of California 
+	! Licensed under the Apache License, Version 2.0 (the "License"); ! you may 
+	not use this file except in compliance with the License. ! you may obtain 
+	a copy of the License from ! ! http://www.apache.org/licenses/LICENSE-2.0 
+	! ! Unless required by applicable law or agreed to in writing, software ! 
+	distributed under the License is distributed on an "AS IS" BASIS, ! WITHOUT 
+	WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ! See the 
+	License for the specific language governing permissions and ! limitations 
+	under the License. ! -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 	<modelVersion>4.0.0</modelVersion>
 	<parent>
 		<artifactId>asterix</artifactId>
@@ -51,13 +47,13 @@
 		</dependency>
 		<dependency>
 			<groupId>edu.uci.ics.asterix</groupId>
-			<artifactId>asterix-external-data</artifactId>
+			<artifactId>asterix-runtime</artifactId>
 			<version>0.8.1-SNAPSHOT</version>
 			<scope>compile</scope>
 		</dependency>
 		<dependency>
 			<groupId>edu.uci.ics.asterix</groupId>
-			<artifactId>asterix-runtime</artifactId>
+			<artifactId>asterix-events</artifactId>
 			<version>0.8.1-SNAPSHOT</version>
 			<scope>compile</scope>
 		</dependency>
@@ -80,6 +76,12 @@
 			<groupId>edu.uci.ics.hyracks</groupId>
 			<artifactId>hyracks-storage-am-lsm-rtree</artifactId>
 		</dependency>
+                <dependency>
+                        <groupId>edu.uci.ics.hyracks</groupId>
+                        <artifactId>hyracks-hdfs-core</artifactId>
+                        <version>${hyracks.version}</version>
+                </dependency>
+
 		<dependency>
 			<groupId>org.apache.hadoop</groupId>
 			<artifactId>hadoop-core</artifactId>
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataCache.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataCache.java
index c078be8..e83c5e0 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataCache.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataCache.java
@@ -26,9 +26,12 @@
 import edu.uci.ics.asterix.metadata.entities.DatasourceAdapter;
 import edu.uci.ics.asterix.metadata.entities.Datatype;
 import edu.uci.ics.asterix.metadata.entities.Dataverse;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity;
+import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
 import edu.uci.ics.asterix.metadata.entities.Function;
 import edu.uci.ics.asterix.metadata.entities.Index;
 import edu.uci.ics.asterix.metadata.entities.NodeGroup;
+import edu.uci.ics.asterix.metadata.feeds.FeedId;
 
 /**
  * Caches metadata entities such that the MetadataManager does not have to
@@ -51,6 +54,11 @@
     protected final Map<FunctionSignature, Function> functions = new HashMap<FunctionSignature, Function>();
     // Key is adapter dataverse. Key of value map is the adapter name  
     protected final Map<String, Map<String, DatasourceAdapter>> adapters = new HashMap<String, Map<String, DatasourceAdapter>>();
+    // Key is FeedId   
+    protected final Map<FeedId, FeedActivity> feedActivity = new HashMap<FeedId, FeedActivity>();
+
+    // Key is DataverseName, Key of the value map is the Policy name   
+    protected final Map<String, Map<String, FeedPolicy>> feedPolicies = new HashMap<String, Map<String, FeedPolicy>>();
 
     // Atomically executes all metadata operations in ctx's log.
     public void commit(MetadataTransactionContext ctx) {
@@ -86,13 +94,16 @@
                         synchronized (datatypes) {
                             synchronized (functions) {
                                 synchronized (adapters) {
-                                    dataverses.clear();
-                                    nodeGroups.clear();
-                                    datasets.clear();
-                                    indexes.clear();
-                                    datatypes.clear();
-                                    functions.clear();
-                                    adapters.clear();
+                                    synchronized (feedActivity) {
+                                        dataverses.clear();
+                                        nodeGroups.clear();
+                                        datasets.clear();
+                                        indexes.clear();
+                                        datatypes.clear();
+                                        functions.clear();
+                                        adapters.clear();
+                                        feedActivity.clear();
+                                    }
                                 }
                             }
                         }
@@ -180,20 +191,34 @@
                 synchronized (indexes) {
                     synchronized (datatypes) {
                         synchronized (functions) {
-                            datasets.remove(dataverse.getDataverseName());
-                            indexes.remove(dataverse.getDataverseName());
-                            datatypes.remove(dataverse.getDataverseName());
-                            adapters.remove(dataverse.getDataverseName());
-                            List<FunctionSignature> markedFunctionsForRemoval = new ArrayList<FunctionSignature>();
-                            for (FunctionSignature signature : functions.keySet()) {
-                                if (signature.getNamespace().equals(dataverse.getDataverseName())) {
-                                    markedFunctionsForRemoval.add(signature);
+                            synchronized (adapters) {
+                                synchronized (feedActivity) {
+                                    datasets.remove(dataverse.getDataverseName());
+                                    indexes.remove(dataverse.getDataverseName());
+                                    datatypes.remove(dataverse.getDataverseName());
+                                    adapters.remove(dataverse.getDataverseName());
+                                    List<FunctionSignature> markedFunctionsForRemoval = new ArrayList<FunctionSignature>();
+                                    for (FunctionSignature signature : functions.keySet()) {
+                                        if (signature.getNamespace().equals(dataverse.getDataverseName())) {
+                                            markedFunctionsForRemoval.add(signature);
+                                        }
+                                    }
+                                    for (FunctionSignature signature : markedFunctionsForRemoval) {
+                                        functions.remove(signature);
+                                    }
+                                    List<FeedId> feedActivitiesMarkedForRemoval = new ArrayList<FeedId>();
+                                    for (FeedId fid : feedActivity.keySet()) {
+                                        if (fid.getDataverse().equals(dataverse.getDataverseName())) {
+                                            feedActivitiesMarkedForRemoval.add(fid);
+                                        }
+                                    }
+                                    for (FeedId fid : feedActivitiesMarkedForRemoval) {
+                                        feedActivity.remove(fid);
+                                    }
+
+                                    return dataverses.remove(dataverse.getDataverseName());
                                 }
                             }
-                            for (FunctionSignature signature : markedFunctionsForRemoval) {
-                                functions.remove(signature);
-                            }
-                            return dataverses.remove(dataverse.getDataverseName());
                         }
                     }
                 }
@@ -374,6 +399,32 @@
         }
     }
 
+    public Object addFeedPolicyIfNotExists(FeedPolicy feedPolicy) {
+        synchronized (feedPolicy) {
+            Map<String, FeedPolicy> p = feedPolicies.get(feedPolicy.getDataverseName());
+            if (p == null) {
+                p = new HashMap<String, FeedPolicy>();
+                p.put(feedPolicy.getPolicyName(), feedPolicy);
+                feedPolicies.put(feedPolicy.getDataverseName(), p);
+            } else {
+                if (p.get(feedPolicy.getPolicyName()) == null) {
+                    p.put(feedPolicy.getPolicyName(), feedPolicy);
+                }
+            }
+            return null;
+        }
+    }
+
+    public Object dropFeedPolicy(FeedPolicy feedPolicy) {
+        synchronized (feedPolicies) {
+            Map<String, FeedPolicy> p = feedPolicies.get(feedPolicy.getDataverseName());
+            if (p != null && p.get(feedPolicy.getPolicyName()) != null) {
+                return p.remove(feedPolicy).getPolicyName();
+            }
+            return null;
+        }
+    }
+
     public Object addAdapterIfNotExists(DatasourceAdapter adapter) {
         synchronized (adapters) {
             Map<String, DatasourceAdapter> adaptersInDataverse = adapters.get(adapter.getAdapterIdentifier().getNamespace());
@@ -399,4 +450,21 @@
             return null;
         }
     }
+
+    public Object addFeedActivityIfNotExists(FeedActivity fa) {
+        synchronized (feedActivity) {
+            FeedId fid = new FeedId(fa.getDataverseName(), fa.getDatasetName());
+            if (!feedActivity.containsKey(fid)) {
+                feedActivity.put(fid, fa);
+            }
+        }
+        return null;
+    }
+
+    public Object dropFeedActivity(FeedActivity fa) {
+        synchronized (feedActivity) {
+            FeedId fid = new FeedId(fa.getDataverseName(), fa.getDatasetName());
+            return feedActivity.remove(fid);
+        }
+    }
 }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataException.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataException.java
index b606839..e58466d 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataException.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataException.java
@@ -17,6 +17,7 @@
 
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
 
+
 public class MetadataException extends AsterixException {
     private static final long serialVersionUID = 1L;
 
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
index 8f0eedb..e02cbba 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
@@ -31,11 +31,15 @@
 import edu.uci.ics.asterix.metadata.entities.DatasourceAdapter;
 import edu.uci.ics.asterix.metadata.entities.Datatype;
 import edu.uci.ics.asterix.metadata.entities.Dataverse;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity;
+import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
 import edu.uci.ics.asterix.metadata.entities.Function;
 import edu.uci.ics.asterix.metadata.entities.Index;
 import edu.uci.ics.asterix.metadata.entities.Node;
 import edu.uci.ics.asterix.metadata.entities.NodeGroup;
+import edu.uci.ics.asterix.metadata.feeds.FeedId;
 import edu.uci.ics.asterix.transaction.management.service.transaction.JobIdFactory;
+import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 
 /**
  * Provides access to Asterix metadata via remote methods to the metadata node.
@@ -77,6 +81,7 @@
     private IMetadataNode metadataNode;
     private final ReadWriteLock metadataLatch;
     private final AsterixMetadataProperties metadataProperties;
+    private IHyracksClientConnection hcc;
 
     public MetadataManager(IAsterixStateProxy proxy, AsterixMetadataProperties metadataProperties) {
         if (proxy == null) {
@@ -95,10 +100,14 @@
             if (metadataNode != null) {
                 return;
             }
-            metadataNode = proxy.getMetadataNode();
-            if (metadataNode == null) {
-                throw new Error("Failed to get the MetadataNode.\n" + "The MetadataNode was configured to run on NC: "
-                        + metadataProperties.getMetadataNodeName());
+            while (metadataNode == null) {
+                metadataNode = proxy.getMetadataNode();
+                try {
+                    Thread.sleep(500);
+                } catch (InterruptedException ie) {
+                    throw new RemoteException("Interrupted while waiting for obtaining handle to Metadata node " + "("
+                            + metadataProperties.getMetadataNodeName() + ")");
+                }
             }
         }
     }
@@ -530,6 +539,16 @@
     }
 
     @Override
+    public void addFeedPolicy(MetadataTransactionContext mdTxnCtx, FeedPolicy feedPolicy) throws MetadataException {
+        try {
+            metadataNode.addFeedPolicy(mdTxnCtx.getJobId(), feedPolicy);
+        } catch (RemoteException e) {
+            throw new MetadataException(e);
+        }
+        mdTxnCtx.addFeedPolicy(feedPolicy);
+    }
+
+    @Override
     public void initializeDatasetIdFactory(MetadataTransactionContext ctx) throws MetadataException {
         try {
             metadataNode.initializeDatasetIdFactory(ctx.getJobId());
@@ -585,6 +604,29 @@
     }
 
     @Override
+    public void registerFeedActivity(MetadataTransactionContext ctx, FeedId feedId, FeedActivity feedActivity)
+            throws MetadataException {
+        try {
+            metadataNode.registerFeedActivity(ctx.getJobId(), feedId, feedActivity);
+        } catch (RemoteException e) {
+            throw new MetadataException(e);
+        }
+    }
+
+    @Override
+    public FeedActivity getRecentFeedActivity(MetadataTransactionContext ctx, String dataverseName, String datasetName)
+            throws MetadataException {
+
+        FeedActivity feedActivity = null;
+        try {
+            feedActivity = metadataNode.getRecentFeedActivity(ctx.getJobId(), new FeedId(dataverseName, datasetName));
+        } catch (RemoteException e) {
+            throw new MetadataException(e);
+        }
+        return feedActivity;
+    }
+
+    @Override
     public void acquireWriteLatch() {
         metadataLatch.writeLock().lock();
     }
@@ -603,4 +645,18 @@
     public void releaseReadLatch() {
         metadataLatch.readLock().unlock();
     }
+
+    @Override
+    public FeedPolicy getFeedPolicy(MetadataTransactionContext ctx, String dataverse, String policyName)
+            throws MetadataException {
+
+        FeedPolicy FeedPolicy = null;
+        try {
+            FeedPolicy = metadataNode.getFeedPolicy(ctx.getJobId(), dataverse, policyName);
+        } catch (RemoteException e) {
+            throw new MetadataException(e);
+        }
+        return FeedPolicy;
+    }
+
 }
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
index 5549c02..7014500 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
@@ -17,6 +17,7 @@
 
 import java.rmi.RemoteException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 import edu.uci.ics.asterix.common.api.IAsterixAppRuntimeContext;
@@ -42,6 +43,8 @@
 import edu.uci.ics.asterix.metadata.entities.DatasourceAdapter;
 import edu.uci.ics.asterix.metadata.entities.Datatype;
 import edu.uci.ics.asterix.metadata.entities.Dataverse;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity;
+import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
 import edu.uci.ics.asterix.metadata.entities.Function;
 import edu.uci.ics.asterix.metadata.entities.Index;
 import edu.uci.ics.asterix.metadata.entities.InternalDatasetDetails;
@@ -51,10 +54,14 @@
 import edu.uci.ics.asterix.metadata.entitytupletranslators.DatasourceAdapterTupleTranslator;
 import edu.uci.ics.asterix.metadata.entitytupletranslators.DatatypeTupleTranslator;
 import edu.uci.ics.asterix.metadata.entitytupletranslators.DataverseTupleTranslator;
+import edu.uci.ics.asterix.metadata.entitytupletranslators.FeedActivityTupleTranslator;
+import edu.uci.ics.asterix.metadata.entitytupletranslators.FeedPolicyTupleTranslator;
 import edu.uci.ics.asterix.metadata.entitytupletranslators.FunctionTupleTranslator;
 import edu.uci.ics.asterix.metadata.entitytupletranslators.IndexTupleTranslator;
 import edu.uci.ics.asterix.metadata.entitytupletranslators.NodeGroupTupleTranslator;
 import edu.uci.ics.asterix.metadata.entitytupletranslators.NodeTupleTranslator;
+import edu.uci.ics.asterix.metadata.feeds.FeedActivityIdFactory;
+import edu.uci.ics.asterix.metadata.feeds.FeedId;
 import edu.uci.ics.asterix.metadata.valueextractors.DatasetNameValueExtractor;
 import edu.uci.ics.asterix.metadata.valueextractors.DatatypeNameValueExtractor;
 import edu.uci.ics.asterix.metadata.valueextractors.MetadataEntityValueExtractor;
@@ -772,6 +779,7 @@
             }
             return results.get(0);
         } catch (Exception e) {
+            e.printStackTrace();
             throw new MetadataException(e);
         }
     }
@@ -1136,4 +1144,98 @@
     public int getMostRecentDatasetId() throws MetadataException, RemoteException {
         return DatasetIdFactory.getMostRecentDatasetId();
     }
+
+    @Override
+    public void registerFeedActivity(JobId jobId, FeedId feedId, FeedActivity feedActivity) throws MetadataException,
+            RemoteException {
+        try {
+            if (!FeedActivityIdFactory.isInitialized()) {
+                initializeFeedActivityIdFactory(jobId);
+            }
+            feedActivity.setActivityId(FeedActivityIdFactory.generateFeedActivityId());
+            FeedActivityTupleTranslator tupleReaderWriter = new FeedActivityTupleTranslator(true);
+            ITupleReference tuple = tupleReaderWriter.getTupleFromMetadataEntity(feedActivity);
+            insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.FEED_ACTIVITY_DATASET, tuple);
+        } catch (Exception e) {
+            throw new MetadataException(e);
+        }
+
+    }
+
+    @Override
+    public FeedActivity getRecentFeedActivity(JobId jobId, FeedId feedId) throws MetadataException, RemoteException {
+        try {
+            ITupleReference searchKey = createTuple(feedId.getDataverse(), feedId.getDataset());
+            FeedActivityTupleTranslator tupleReaderWriter = new FeedActivityTupleTranslator(false);
+            List<FeedActivity> results = new ArrayList<FeedActivity>();
+            IValueExtractor<FeedActivity> valueExtractor = new MetadataEntityValueExtractor<FeedActivity>(
+                    tupleReaderWriter);
+            searchIndex(jobId, MetadataPrimaryIndexes.FEED_ACTIVITY_DATASET, searchKey, valueExtractor, results);
+            if (!results.isEmpty()) {
+                Collections.sort(results);
+                return results.get(results.size() - 1);
+            }
+            return null;
+        } catch (Exception e) {
+            throw new MetadataException(e);
+        }
+    }
+
+    @Override
+    public void initializeFeedActivityIdFactory(JobId jobId) throws MetadataException, RemoteException {
+        try {
+            ITupleReference searchKey = createTuple();
+            FeedActivityTupleTranslator tupleReaderWriter = new FeedActivityTupleTranslator(true);
+            List<FeedActivity> results = new ArrayList<FeedActivity>();
+            IValueExtractor<FeedActivity> valueExtractor = new MetadataEntityValueExtractor<FeedActivity>(
+                    tupleReaderWriter);
+            searchIndex(jobId, MetadataPrimaryIndexes.FEED_ACTIVITY_DATASET, searchKey, valueExtractor, results);
+            int maxActivityId = 0;
+            for (FeedActivity fa : results) {
+                if (maxActivityId < fa.getActivityId()) {
+                    maxActivityId = fa.getActivityId();
+                }
+            }
+            FeedActivityIdFactory.initialize(maxActivityId);
+        } catch (Exception e) {
+            throw new MetadataException(e);
+        }
+
+    }
+
+    @Override
+    public void addFeedPolicy(JobId jobId, FeedPolicy feedPolicy) throws MetadataException, RemoteException {
+        try {
+            // Insert into the 'FeedPolicy' dataset.
+            FeedPolicyTupleTranslator tupleReaderWriter = new FeedPolicyTupleTranslator(true);
+            ITupleReference feedPolicyTuple = tupleReaderWriter.getTupleFromMetadataEntity(feedPolicy);
+            insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.FEED_POLICY_DATASET, feedPolicyTuple);
+
+        } catch (BTreeDuplicateKeyException e) {
+            throw new MetadataException("A feed policy with this name " + feedPolicy.getPolicyName()
+                    + " already exists in dataverse '" + feedPolicy.getPolicyName() + "'.", e);
+        } catch (Exception e) {
+            throw new MetadataException(e);
+        }
+
+    }
+
+    @Override
+    public FeedPolicy getFeedPolicy(JobId jobId, String dataverse, String policyName) throws MetadataException,
+            RemoteException {
+
+        try {
+            ITupleReference searchKey = createTuple(dataverse, policyName);
+            FeedPolicyTupleTranslator tupleReaderWriter = new FeedPolicyTupleTranslator(false);
+            List<FeedPolicy> results = new ArrayList<FeedPolicy>();
+            IValueExtractor<FeedPolicy> valueExtractor = new MetadataEntityValueExtractor<FeedPolicy>(tupleReaderWriter);
+            searchIndex(jobId, MetadataPrimaryIndexes.FEED_POLICY_DATASET, searchKey, valueExtractor, results);
+            if (!results.isEmpty()) {
+                return results.get(0);
+            }
+            return null;
+        } catch (Exception e) {
+            throw new MetadataException(e);
+        }
+    }
 }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataTransactionContext.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataTransactionContext.java
index dedcedf..4e741c3 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataTransactionContext.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataTransactionContext.java
@@ -19,15 +19,16 @@
 
 import edu.uci.ics.asterix.common.functions.FunctionSignature;
 import edu.uci.ics.asterix.common.transactions.JobId;
-import edu.uci.ics.asterix.external.dataset.adapter.AdapterIdentifier;
 import edu.uci.ics.asterix.metadata.api.IMetadataEntity;
 import edu.uci.ics.asterix.metadata.entities.Dataset;
 import edu.uci.ics.asterix.metadata.entities.DatasourceAdapter;
 import edu.uci.ics.asterix.metadata.entities.Datatype;
 import edu.uci.ics.asterix.metadata.entities.Dataverse;
+import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
 import edu.uci.ics.asterix.metadata.entities.Function;
 import edu.uci.ics.asterix.metadata.entities.Index;
 import edu.uci.ics.asterix.metadata.entities.NodeGroup;
+import edu.uci.ics.asterix.metadata.feeds.AdapterIdentifier;
 
 /**
  * Used to implement serializable transactions against the MetadataCache.
@@ -200,4 +201,10 @@
         droppedCache.clear();
         opLog.clear();
     }
+
+    public void addFeedPolicy(FeedPolicy feedPolicy) {
+        droppedCache.dropFeedPolicy(feedPolicy);
+        logAndApply(new MetadataLogicalOperation(feedPolicy, true));
+
+    }
 }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IClusterManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IClusterManager.java
new file mode 100644
index 0000000..386d89e
--- /dev/null
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IClusterManager.java
@@ -0,0 +1,20 @@
+package edu.uci.ics.asterix.metadata.api;
+
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.event.schema.cluster.Node;
+
+public interface IClusterManager {
+
+    /**
+     * @param node
+     * @throws AsterixException
+     */
+    public void addNode(Node node) throws AsterixException;
+
+    /**
+     * @param node
+     * @throws AsterixException
+     */
+    public void removeNode(Node node) throws AsterixException;
+
+}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataManager.java
index 22c5e46..5f1acf8 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataManager.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataManager.java
@@ -26,10 +26,13 @@
 import edu.uci.ics.asterix.metadata.entities.DatasourceAdapter;
 import edu.uci.ics.asterix.metadata.entities.Datatype;
 import edu.uci.ics.asterix.metadata.entities.Dataverse;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity;
+import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
 import edu.uci.ics.asterix.metadata.entities.Function;
 import edu.uci.ics.asterix.metadata.entities.Index;
 import edu.uci.ics.asterix.metadata.entities.Node;
 import edu.uci.ics.asterix.metadata.entities.NodeGroup;
+import edu.uci.ics.asterix.metadata.feeds.FeedId;
 
 /**
  * A metadata manager provides user access to Asterix metadata (e.g., types,
@@ -127,7 +130,7 @@
      * @throws MetadataException
      */
     List<Dataverse> getDataverses(MetadataTransactionContext ctx) throws MetadataException;
-    
+
     /**
      * Retrieves a dataverse with given name.
      * 
@@ -178,7 +181,7 @@
      *             For example, if the dataset already exists.
      */
     public void addDataset(MetadataTransactionContext ctx, Dataset dataset) throws MetadataException;
-    
+
     /**
      * Retrieves a dataset within a given dataverse.
      * 
@@ -440,10 +443,39 @@
     public List<Function> getDataverseFunctions(MetadataTransactionContext ctx, String dataverseName)
             throws MetadataException;
 
+    /**
+     * @param ctx
+     * @param feedId
+     * @param feedActivity
+     * @throws MetadataException
+     */
+    public void registerFeedActivity(MetadataTransactionContext ctx, FeedId feedId, FeedActivity feedActivity)
+            throws MetadataException;
+
+    /**
+     * @param ctx
+     * @param dataverseName
+     * @param datasetName
+     * @return
+     * @throws MetadataException
+     */
+    public FeedActivity getRecentFeedActivity(MetadataTransactionContext ctx, String dataverseName, String datasetName)
+            throws MetadataException;
+
+    /**
+     * @param ctx
+     * @param policy
+     * @throws MetadataException
+     */
+    public void addFeedPolicy(MetadataTransactionContext ctx, FeedPolicy policy) throws MetadataException;
+
+    public FeedPolicy getFeedPolicy(MetadataTransactionContext ctx, String dataverse, String policyName)
+            throws MetadataException;
+
     public void initializeDatasetIdFactory(MetadataTransactionContext ctx) throws MetadataException;
-    
+
     public int getMostRecentDatasetId() throws MetadataException;
-    
+
     public void acquireWriteLatch();
 
     public void releaseWriteLatch();
@@ -452,5 +484,4 @@
 
     public void releaseReadLatch();
 
-
 }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataNode.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataNode.java
index d1e63e1..cf5f41e 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataNode.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataNode.java
@@ -28,10 +28,13 @@
 import edu.uci.ics.asterix.metadata.entities.DatasourceAdapter;
 import edu.uci.ics.asterix.metadata.entities.Datatype;
 import edu.uci.ics.asterix.metadata.entities.Dataverse;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity;
+import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
 import edu.uci.ics.asterix.metadata.entities.Function;
 import edu.uci.ics.asterix.metadata.entities.Index;
 import edu.uci.ics.asterix.metadata.entities.Node;
 import edu.uci.ics.asterix.metadata.entities.NodeGroup;
+import edu.uci.ics.asterix.metadata.feeds.FeedId;
 
 /**
  * A metadata node stores metadata in its local storage structures (currently
@@ -383,7 +386,6 @@
     public void addNode(JobId jobId, Node node) throws MetadataException, RemoteException;
 
     /**
-
      * @param jobId
      *            A globally unique id for an active metadata transaction.
      * @param functionSignature
@@ -408,7 +410,8 @@
      *             group to be deleted.
      * @throws RemoteException
      */
-    public void dropFunction(JobId jobId, FunctionSignature functionSignature) throws MetadataException, RemoteException;
+    public void dropFunction(JobId jobId, FunctionSignature functionSignature) throws MetadataException,
+            RemoteException;
 
     /**
      * @param jobId
@@ -442,8 +445,8 @@
     public List<DatasourceAdapter> getDataverseAdapters(JobId jobId, String dataverseName) throws MetadataException,
             RemoteException;
 
-    public DatasourceAdapter getAdapter(JobId jobId, String dataverseName, String adapterName) throws MetadataException,
-            RemoteException;
+    public DatasourceAdapter getAdapter(JobId jobId, String dataverseName, String adapterName)
+            throws MetadataException, RemoteException;
 
     /**
      * Deletes a adapter , acquiring local locks on behalf of the given
@@ -472,8 +475,53 @@
      */
     public void addAdapter(JobId jobId, DatasourceAdapter adapter) throws MetadataException, RemoteException;
 
+    /**
+     * @param jobId
+     * @param feedId
+     * @return
+     * @throws MetadataException
+     * @throws RemoteException
+     */
+    public FeedActivity getRecentFeedActivity(JobId jobId, FeedId feedId) throws MetadataException, RemoteException;
+
     public void initializeDatasetIdFactory(JobId jobId) throws MetadataException, RemoteException;
-    
+
+    /**
+     * @param jobId
+     * @throws MetadataException
+     * @throws RemoteException
+     */
+    public void initializeFeedActivityIdFactory(JobId jobId) throws MetadataException, RemoteException;
+
     public int getMostRecentDatasetId() throws MetadataException, RemoteException;
 
+    /**
+     * @param jobId
+     *            A globally unique id for an active metadata transaction.
+     * @param feedId
+     *            A unique id for the feed
+     * @param feedActivity
+     */
+    public void registerFeedActivity(JobId jobId, FeedId feedId, FeedActivity feedActivity) throws MetadataException,
+            RemoteException;
+
+    /**
+     * @param jobId
+     * @param feedPolicy
+     * @throws MetadataException
+     * @throws RemoteException
+     */
+    public void addFeedPolicy(JobId jobId, FeedPolicy feedPolicy) throws MetadataException, RemoteException;
+
+    /**
+     * @param jobId
+     * @param dataverse
+     * @param policy
+     * @return
+     * @throws MetadataException
+     * @throws RemoteException
+     */
+    public FeedPolicy getFeedPolicy(JobId jobId, String dataverse, String policy) throws MetadataException,
+            RemoteException;
+
 }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
index 715448d..d31fa11 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -37,8 +37,6 @@
 import edu.uci.ics.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
 import edu.uci.ics.asterix.common.transactions.IResourceManager.ResourceType;
 import edu.uci.ics.asterix.common.transactions.TransactionalResourceManagerRepository;
-import edu.uci.ics.asterix.external.adapter.factory.IAdapterFactory;
-import edu.uci.ics.asterix.external.dataset.adapter.AdapterIdentifier;
 import edu.uci.ics.asterix.metadata.IDatasetDetails;
 import edu.uci.ics.asterix.metadata.MetadataException;
 import edu.uci.ics.asterix.metadata.MetadataManager;
@@ -50,12 +48,16 @@
 import edu.uci.ics.asterix.metadata.entities.DatasourceAdapter;
 import edu.uci.ics.asterix.metadata.entities.Datatype;
 import edu.uci.ics.asterix.metadata.entities.Dataverse;
+import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
 import edu.uci.ics.asterix.metadata.entities.Index;
 import edu.uci.ics.asterix.metadata.entities.InternalDatasetDetails;
 import edu.uci.ics.asterix.metadata.entities.InternalDatasetDetails.FileStructure;
 import edu.uci.ics.asterix.metadata.entities.InternalDatasetDetails.PartitioningStrategy;
 import edu.uci.ics.asterix.metadata.entities.Node;
 import edu.uci.ics.asterix.metadata.entities.NodeGroup;
+import edu.uci.ics.asterix.metadata.feeds.AdapterIdentifier;
+import edu.uci.ics.asterix.metadata.feeds.BuiltinFeedPolicies;
+import edu.uci.ics.asterix.metadata.feeds.IAdapterFactory;
 import edu.uci.ics.asterix.om.types.BuiltinType;
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.asterix.runtime.formats.NonTaggedDataFormat;
@@ -118,7 +120,8 @@
                 MetadataPrimaryIndexes.DATASET_DATASET, MetadataPrimaryIndexes.DATATYPE_DATASET,
                 MetadataPrimaryIndexes.INDEX_DATASET, MetadataPrimaryIndexes.NODE_DATASET,
                 MetadataPrimaryIndexes.NODEGROUP_DATASET, MetadataPrimaryIndexes.FUNCTION_DATASET,
-                MetadataPrimaryIndexes.DATASOURCE_ADAPTER_DATASET };
+                MetadataPrimaryIndexes.DATASOURCE_ADAPTER_DATASET, MetadataPrimaryIndexes.FEED_ACTIVITY_DATASET,
+                MetadataPrimaryIndexes.FEED_POLICY_DATASET };
         secondaryIndexes = new IMetadataIndex[] { MetadataSecondaryIndexes.GROUPNAME_ON_DATASET_INDEX,
                 MetadataSecondaryIndexes.DATATYPENAME_ON_DATASET_INDEX,
                 MetadataSecondaryIndexes.DATATYPENAME_ON_DATATYPE_INDEX };
@@ -179,6 +182,7 @@
                 insertNodes(mdTxnCtx);
                 insertInitialGroups(mdTxnCtx);
                 insertInitialAdapters(mdTxnCtx);
+                insertInitialFeedPolicies(mdTxnCtx);
 
                 if (LOGGER.isLoggable(Level.INFO)) {
                     LOGGER.info("Finished creating metadata B-trees.");
@@ -321,6 +325,12 @@
         }
     }
 
+    private static void insertInitialFeedPolicies(MetadataTransactionContext mdTxnCtx) throws Exception {
+        for (FeedPolicy feedPolicy : BuiltinFeedPolicies.policies) {
+            MetadataManager.INSTANCE.addFeedPolicy(mdTxnCtx, feedPolicy);
+        }
+    }
+
     private static DatasourceAdapter getAdapter(String adapterFactoryClassName) throws Exception {
         String adapterName = ((IAdapterFactory) (Class.forName(adapterFactoryClassName).newInstance())).getName();
         return new DatasourceAdapter(new AdapterIdentifier(MetadataConstants.METADATA_DATAVERSE_NAME, adapterName),
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataPrimaryIndexes.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataPrimaryIndexes.java
index 8bdd92b..90c75ec 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataPrimaryIndexes.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataPrimaryIndexes.java
@@ -32,6 +32,10 @@
     public static IMetadataIndex NODE_DATASET;
     public static IMetadataIndex NODEGROUP_DATASET;
     public static IMetadataIndex FUNCTION_DATASET;
+    public static IMetadataIndex DATASOURCE_ADAPTER_DATASET;
+    public static IMetadataIndex FEED_ACTIVITY_DATASET;
+    public static IMetadataIndex FEED_POLICY_DATASET;
+
     public static final int METADATA_DATASET_ID = 0;
     public static final int DATAVERSE_DATASET_ID = 1;
     public static final int DATASET_DATASET_ID = 2;
@@ -41,9 +45,10 @@
     public static final int NODEGROUP_DATASET_ID = 6;
     public static final int FUNCTION_DATASET_ID = 7;
     public static final int DATASOURCE_ADAPTER_DATASET_ID = 8;
-    public static final int FIRST_AVAILABLE_USER_DATASET_ID = 100;
+    public static final int FEED_ACTIVITY_DATASET_ID = 9;
+    public static final int FEED_POLICY_DATASET_ID = 10;
 
-    public static IMetadataIndex DATASOURCE_ADAPTER_DATASET;
+    public static final int FIRST_AVAILABLE_USER_DATASET_ID = 100;
 
     /**
      * Create all metadata primary index descriptors. MetadataRecordTypes must
@@ -92,5 +97,15 @@
                 BuiltinType.ASTRING, BuiltinType.ASTRING }, new String[] { "DataverseName", "Name" }, 0,
                 MetadataRecordTypes.DATASOURCE_ADAPTER_RECORDTYPE, DATASOURCE_ADAPTER_DATASET_ID, true, new int[] { 0,
                         1 });
+
+        FEED_ACTIVITY_DATASET = new MetadataIndex("FeedActivity", null, 4, new IAType[] { BuiltinType.ASTRING,
+                BuiltinType.ASTRING, BuiltinType.AINT32 },
+                new String[] { "DataverseName", "DatasetName", "ActivityId" }, 0,
+                MetadataRecordTypes.FEED_ACTIVITY_RECORDTYPE, FEED_ACTIVITY_DATASET_ID, true, new int[] { 0, 1, 2 });
+
+        FEED_POLICY_DATASET = new MetadataIndex("FeedPolicy", null, 3, new IAType[] { BuiltinType.ASTRING,
+                BuiltinType.ASTRING }, new String[] { "DataverseName", "PolicyName" }, 0,
+                MetadataRecordTypes.FEED_POLICY_RECORDTYPE, FEED_POLICY_DATASET_ID, true, new int[] { 0, 1 });
+
     }
 }
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataRecordTypes.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataRecordTypes.java
index 50681ee..f922169 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataRecordTypes.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataRecordTypes.java
@@ -47,6 +47,10 @@
     public static ARecordType NODEGROUP_RECORDTYPE;
     public static ARecordType FUNCTION_RECORDTYPE;
     public static ARecordType DATASOURCE_ADAPTER_RECORDTYPE;
+    public static ARecordType FEED_ACTIVITY_RECORDTYPE;
+    public static ARecordType FEED_POLICY_RECORDTYPE;
+    public static ARecordType POLICY_PARAMS_RECORDTYPE;
+    public static ARecordType FEED_ACTIVITY_DETAILS_RECORDTYPE;
 
     /**
      * Create all metadata record types.
@@ -56,6 +60,7 @@
         // depend on other types being created first.
         // These calls are one "dependency chain".
         try {
+            POLICY_PARAMS_RECORDTYPE = createPropertiesRecordType();
             DATASOURCE_ADAPTER_PROPERTIES_RECORDTYPE = createPropertiesRecordType();
             INTERNAL_DETAILS_RECORDTYPE = createInternalDetailsRecordType();
             EXTERNAL_DETAILS_RECORDTYPE = createExternalDetailsRecordType();
@@ -76,11 +81,26 @@
             NODEGROUP_RECORDTYPE = createNodeGroupRecordType();
             FUNCTION_RECORDTYPE = createFunctionRecordType();
             DATASOURCE_ADAPTER_RECORDTYPE = createDatasourceAdapterRecordType();
+            FEED_ACTIVITY_DETAILS_RECORDTYPE = createPropertiesRecordType();
+            FEED_ACTIVITY_RECORDTYPE = createFeedActivityRecordType();
+            FEED_POLICY_RECORDTYPE = createFeedPolicyRecordType();
         } catch (AsterixException e) {
             throw new MetadataException(e);
         }
     }
 
+    public static final int FEED_POLICY_ARECORD_DATAVERSE_NAME_FIELD_INDEX = 0;
+    public static final int FEED_POLICY_ARECORD_POLICY_NAME_FIELD_INDEX = 1;
+    public static final int FEED_POLICY_ARECORD_DESCRIPTION_FIELD_INDEX = 2;
+    public static final int FEED_POLICY_ARECORD_PROPERTIES_FIELD_INDEX = 3;
+
+    private static ARecordType createFeedPolicyRecordType() throws AsterixException {
+        AUnorderedListType listPropertiesType = new AUnorderedListType(POLICY_PARAMS_RECORDTYPE, null);
+        String[] fieldNames = { "DataverseName", "PolicyName", "Description", "Properties" };
+        IAType[] fieldTypes = { BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.ASTRING, listPropertiesType };
+        return new ARecordType("FeedPolicyRecordType", fieldNames, fieldTypes, true);
+    }
+
     // Helper constants for accessing fields in an ARecord of type
     // DataverseRecordType.
     public static final int DATAVERSE_ARECORD_NAME_FIELD_INDEX = 0;
@@ -97,8 +117,8 @@
     // Helper constants for accessing fields in an ARecord of anonymous type
     // dataset properties.
     // Used for dataset hints or dataset adapter properties.
-    public static final int DATASOURCE_PROPERTIES_NAME_FIELD_INDEX = 0;
-    public static final int DATASOURCE_PROPERTIES_VALUE_FIELD_INDEX = 1;
+    public static final int PROPERTIES_NAME_FIELD_INDEX = 0;
+    public static final int PROPERTIES_VALUE_FIELD_INDEX = 1;
 
     private static final ARecordType createPropertiesRecordType() throws AsterixException {
         String[] fieldNames = { "Name", "Value" };
@@ -143,13 +163,17 @@
     public static final int FEED_DETAILS_ARECORD_PROPERTIES_FIELD_INDEX = 6;
     public static final int FEED_DETAILS_ARECORD_FUNCTION_FIELD_INDEX = 7;
     public static final int FEED_DETAILS_ARECORD_STATE_FIELD_INDEX = 8;
+    public static final int FEED_DETAILS_ARECORD_INGEST_NODES_FIELD_INDEX = 9;
+    public static final int FEED_DETAILS_ARECORD_COMPUTE_NODES_FIELD_INDEX = 10;
 
     private static final ARecordType createFeedDetailsRecordType() throws AsterixException {
         AOrderedListType orderedListType = new AOrderedListType(BuiltinType.ASTRING, null);
         AOrderedListType orderedListOfPropertiesType = new AOrderedListType(DATASOURCE_ADAPTER_PROPERTIES_RECORDTYPE,
                 null);
+        AUnorderedListType unorderedListType = new AUnorderedListType(BuiltinType.ASTRING, null);
+
         String[] fieldNames = { "FileStructure", "PartitioningStrategy", "PartitioningKey", "PrimaryKey", "GroupName",
-                "DatasourceAdapter", "Properties", "Function", "Status" };
+                "DatasourceAdapter", "Properties", "Function", "Status", "IngestNodes", "ComputeNodes" };
 
         List<IAType> feedFunctionUnionList = new ArrayList<IAType>();
         feedFunctionUnionList.add(BuiltinType.ANULL);
@@ -158,7 +182,7 @@
 
         IAType[] fieldTypes = { BuiltinType.ASTRING, BuiltinType.ASTRING, orderedListType, orderedListType,
                 BuiltinType.ASTRING, BuiltinType.ASTRING, orderedListOfPropertiesType, feedFunctionUnion,
-                BuiltinType.ASTRING };
+                BuiltinType.ASTRING, unorderedListType, unorderedListType };
 
         return new ARecordType(null, fieldNames, fieldTypes, true);
     }
@@ -357,4 +381,22 @@
         return new ARecordType("DatasourceAdapterRecordType", fieldNames, fieldTypes, true);
     }
 
+    // Helper constants for accessing fields in an ARecord of type
+    // FeedActivityRecordType.
+    public static final int FEED_ACTIVITY_ARECORD_DATAVERSE_NAME_FIELD_INDEX = 0;
+    public static final int FEED_ACTIVITY_ARECORD_DATASET_NAME_FIELD_INDEX = 1;
+    public static final int FEED_ACTIVITY_ARECORD_ACTIVITY_ID_FIELD_INDEX = 2;
+    public static final int FEED_ACTIVITY_ARECORD_ACTIVITY_TYPE_FIELD_INDEX = 3;
+    public static final int FEED_ACTIVITY_ARECORD_LAST_UPDATE_TIMESTAMP_FIELD_INDEX = 4;
+    public static final int FEED_ACTIVITY_ARECORD_DETAILS_FIELD_INDEX = 5;
+
+    private static ARecordType createFeedActivityRecordType() throws AsterixException {
+        AUnorderedListType unorderedPropertyListType = new AUnorderedListType(FEED_ACTIVITY_DETAILS_RECORDTYPE, null);
+        String[] fieldNames = { "DataverseName", "DatasetName", "ActivityId", "ActivityType", "UpdateTimestamp",
+                "Details" };
+        IAType[] fieldTypes = { BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.AINT32, BuiltinType.ASTRING,
+                BuiltinType.ASTRING, unorderedPropertyListType };
+        return new ARecordType("FeedActivityRecordType", fieldNames, fieldTypes, true);
+    }
+
 }
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/cluster/ClusterManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/cluster/ClusterManager.java
new file mode 100644
index 0000000..b73b861
--- /dev/null
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/cluster/ClusterManager.java
@@ -0,0 +1,85 @@
+package edu.uci.ics.asterix.metadata.cluster;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.Unmarshaller;
+
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.event.management.AsterixEventServiceClient;
+import edu.uci.ics.asterix.event.schema.cluster.Cluster;
+import edu.uci.ics.asterix.event.schema.cluster.Node;
+import edu.uci.ics.asterix.event.schema.pattern.Pattern;
+import edu.uci.ics.asterix.event.schema.pattern.Patterns;
+import edu.uci.ics.asterix.event.service.AsterixEventService;
+import edu.uci.ics.asterix.event.util.PatternCreator;
+import edu.uci.ics.asterix.installer.schema.conf.Configuration;
+import edu.uci.ics.asterix.metadata.api.IClusterManager;
+import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
+import edu.uci.ics.asterix.om.util.AsterixClusterProperties;
+
+public class ClusterManager implements IClusterManager {
+
+    public static ClusterManager INSTANCE = new ClusterManager();
+
+    private static String eventsDir = System.getenv("user.dir") + File.separator + "eventrix";
+
+    private static AsterixEventServiceClient client;
+
+    private ClusterManager() {
+        Cluster asterixCluster = AsterixClusterProperties.INSTANCE.getCluster();
+        String asterixDir = System.getProperty("user.dir") + File.separator + "asterix";
+        String eventHome = asterixCluster.getWorkingDir().getDir();
+        File configFile = new File(System.getProperty("user.dir") + File.separator + "configuration.xml");
+        Configuration configuration = null;
+
+        try {
+            JAXBContext configCtx = JAXBContext.newInstance(Configuration.class);
+            Unmarshaller unmarshaller = configCtx.createUnmarshaller();
+            configuration = (Configuration) unmarshaller.unmarshal(configFile);
+            AsterixEventService.initialize(configuration, asterixDir, eventHome);
+            client = AsterixEventService.getAsterixEventServiceClient(AsterixClusterProperties.INSTANCE.getCluster());
+        } catch (Exception e) {
+            throw new IllegalStateException("Unable to initialize cluster manager" + e);
+        }
+    }
+
+    @Override
+    public void addNode(Node node) throws AsterixException {
+        try {
+            Cluster cluster = AsterixClusterProperties.INSTANCE.getCluster();
+            List<Pattern> pattern = new ArrayList<Pattern>();
+            String asterixInstanceName = AsterixAppContextInfo.getInstance().getMetadataProperties().getInstanceName();
+            Patterns prepareNode = PatternCreator.INSTANCE.createPrepareNodePattern(asterixInstanceName,
+                    AsterixClusterProperties.INSTANCE.getCluster(), node);
+            cluster.getNode().add(node);
+            client.submit(prepareNode);
+
+            pattern.clear();
+            String ccHost = cluster.getMasterNode().getClusterIp();
+            String hostId = node.getId();
+            String nodeControllerId = asterixInstanceName + "_" + node.getId();
+            String iodevices = node.getIodevices() == null ? cluster.getIodevices() : node.getIodevices();
+            Pattern startNC = PatternCreator.INSTANCE.createNCStartPattern(ccHost, hostId, nodeControllerId, iodevices);
+            pattern.add(startNC);
+            Patterns startNCPattern = new Patterns(pattern);
+            client.submit(startNCPattern);
+        } catch (Exception e) {
+            throw new AsterixException(e);
+        }
+
+    }
+
+    @Override
+    public void removeNode(Node node) throws AsterixException {
+
+    }
+
+    private List<Pattern> getRemoveNodePattern(Node node) {
+        List<Pattern> pattern = new ArrayList<Pattern>();
+
+        return pattern;
+    }
+}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/dataset/hints/IHint.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/dataset/hints/IHint.java
index d4a937d..519c831 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/dataset/hints/IHint.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/dataset/hints/IHint.java
@@ -16,6 +16,7 @@
 
 import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
 
+
 /**
  * Represents a hint provided as part of an AQL statement.
  */
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlDataSource.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlDataSource.java
index 0ed3c78..e3e4cb7 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlDataSource.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlDataSource.java
@@ -16,8 +16,11 @@
 package edu.uci.ics.asterix.metadata.declared;
 
 import java.io.IOException;
+import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
@@ -51,6 +54,7 @@
     private IAType[] schemaTypes;
     private INodeDomain domain;
     private AqlDataSourceType datasourceType;
+    private Map<String, Serializable> properties = new HashMap<String, Serializable>();
 
     public enum AqlDataSourceType {
         INTERNAL,
@@ -268,4 +272,12 @@
         return datasourceType;
     }
 
+    public Map<String, Serializable> getProperties() {
+        return properties;
+    }
+
+    public void setProperties(Map<String, Serializable> properties) {
+        this.properties = properties;
+    }
+
 }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
index 51ab213..b2ccbc0 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
@@ -37,17 +37,6 @@
 import edu.uci.ics.asterix.common.transactions.IResourceManager.ResourceType;
 import edu.uci.ics.asterix.common.transactions.JobId;
 import edu.uci.ics.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveValueProviderFactory;
-import edu.uci.ics.asterix.external.adapter.factory.HDFSAdapterFactory;
-import edu.uci.ics.asterix.external.adapter.factory.IAdapterFactory;
-import edu.uci.ics.asterix.external.adapter.factory.IGenericDatasetAdapterFactory;
-import edu.uci.ics.asterix.external.adapter.factory.ITypedDatasetAdapterFactory;
-import edu.uci.ics.asterix.external.data.operator.ExternalDataScanOperatorDescriptor;
-import edu.uci.ics.asterix.external.data.operator.FeedIntakeOperatorDescriptor;
-import edu.uci.ics.asterix.external.data.operator.FeedMessageOperatorDescriptor;
-import edu.uci.ics.asterix.external.dataset.adapter.IDatasourceAdapter;
-import edu.uci.ics.asterix.external.dataset.adapter.ITypedDatasourceAdapter;
-import edu.uci.ics.asterix.external.feed.lifecycle.FeedId;
-import edu.uci.ics.asterix.external.feed.lifecycle.IFeedMessage;
 import edu.uci.ics.asterix.formats.base.IDataFormat;
 import edu.uci.ics.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
 import edu.uci.ics.asterix.formats.nontagged.AqlTypeTraitProvider;
@@ -61,9 +50,23 @@
 import edu.uci.ics.asterix.metadata.entities.Datatype;
 import edu.uci.ics.asterix.metadata.entities.Dataverse;
 import edu.uci.ics.asterix.metadata.entities.ExternalDatasetDetails;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity.FeedActivityDetails;
 import edu.uci.ics.asterix.metadata.entities.FeedDatasetDetails;
+import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
 import edu.uci.ics.asterix.metadata.entities.Index;
 import edu.uci.ics.asterix.metadata.entities.InternalDatasetDetails;
+import edu.uci.ics.asterix.metadata.feeds.BuiltinFeedPolicies;
+import edu.uci.ics.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
+import edu.uci.ics.asterix.metadata.feeds.FeedId;
+import edu.uci.ics.asterix.metadata.feeds.FeedIntakeOperatorDescriptor;
+import edu.uci.ics.asterix.metadata.feeds.FeedMessageOperatorDescriptor;
+import edu.uci.ics.asterix.metadata.feeds.IAdapterFactory;
+import edu.uci.ics.asterix.metadata.feeds.IDatasourceAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IFeedMessage;
+import edu.uci.ics.asterix.metadata.feeds.IGenericDatasetAdapterFactory;
+import edu.uci.ics.asterix.metadata.feeds.ITypedDatasetAdapterFactory;
+import edu.uci.ics.asterix.metadata.feeds.ITypedDatasourceAdapter;
 import edu.uci.ics.asterix.metadata.utils.DatasetUtils;
 import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
 import edu.uci.ics.asterix.om.types.ARecordType;
@@ -106,6 +109,7 @@
 import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
 import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
 import edu.uci.ics.hyracks.algebricks.runtime.operators.std.SinkWriterRuntimeFactory;
+import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 import edu.uci.ics.hyracks.api.context.ICCContext;
 import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
 import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -156,6 +160,7 @@
     private boolean asyncResults;
     private ResultSetId resultSetId;
     private IResultSerializerFactoryProvider resultSerializerFactoryProvider;
+    private IHyracksClientConnection hcc;
 
     private final Dataverse defaultDataverse;
     private JobId jobId;
@@ -164,6 +169,8 @@
 
     private static final Map<String, String> adapterFactoryMapping = initializeAdapterFactoryMapping();
     private static Scheduler hdfsScheduler;
+    public static final String CLUSTER_LOCATIONS = "cluster-locations";
+    public static transient String SCHEDULER = "hdfs-scheduler";
 
     public String getPropertyValue(String propertyName) {
         return config.get(propertyName);
@@ -333,7 +340,7 @@
         IAType itemType = MetadataManager.INSTANCE.getDatatype(mdTxnCtx, dataset.getDataverseName(), itemTypeName)
                 .getDatatype();
         if (dataSource instanceof ExternalFeedDataSource) {
-            return buildFeedIntakeRuntime(jobSpec, dataset);
+            return buildFeedIntakeRuntime(jobSpec, dataset, dataSource);
         } else {
             return buildExternalDataScannerRuntime(jobSpec, itemType,
                     (ExternalDatasetDetails) dataset.getDatasetDetails(), NonTaggedDataFormat.INSTANCE);
@@ -423,7 +430,7 @@
 
     @SuppressWarnings("rawtypes")
     public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildFeedIntakeRuntime(JobSpecification jobSpec,
-            Dataset dataset) throws AlgebricksException {
+            Dataset dataset, IDataSource<AqlSourceId> dataSource) throws AlgebricksException {
 
         FeedDatasetDetails datasetDetails = (FeedDatasetDetails) dataset.getDatasetDetails();
         DatasourceAdapter adapterEntity;
@@ -475,10 +482,16 @@
                 .getSerializerDeserializer(adapterOutputType);
         RecordDescriptor feedDesc = new RecordDescriptor(new ISerializerDeserializer[] { payloadSerde });
 
+        FeedPolicy feedPolicy = (FeedPolicy) ((AqlDataSource) dataSource).getProperties().get(
+                BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY);
+
+        // public FeedIntakeOperatorDescriptor(JobSpecification spec, FeedId feedId, String adapter,
+        //        Map<String, Object> arguments, ARecordType atype, RecordDescriptor rDesc, Map<String, String> feedPolicy) {
+
         FeedIntakeOperatorDescriptor feedIngestor = new FeedIntakeOperatorDescriptor(jobSpec, new FeedId(
                 dataset.getDataverseName(), dataset.getDatasetName()), adapterFactoryClassname,
                 this.wrapPropertiesEmpty(datasetDetails.getProperties()), (ARecordType) adapterOutputType, feedDesc,
-                adapterFactory);
+                feedPolicy.getProperties());
 
         AlgebricksPartitionConstraint constraint = null;
         try {
@@ -491,12 +504,13 @@
 
     public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildFeedMessengerRuntime(
             AqlMetadataProvider metadataProvider, JobSpecification jobSpec, FeedDatasetDetails datasetDetails,
-            String dataverse, String dataset, List<IFeedMessage> feedMessages) throws AlgebricksException {
-        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> spPc = metadataProvider
-                .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverse, dataset, dataset);
+            String dataverse, String dataset, List<IFeedMessage> feedMessages, FeedActivity feedActivity)
+            throws AlgebricksException {
+        AlgebricksPartitionConstraint partitionConstraint = new AlgebricksAbsolutePartitionConstraint(feedActivity
+                .getFeedActivityDetails().get(FeedActivityDetails.INGEST_LOCATIONS).split(","));
         FeedMessageOperatorDescriptor feedMessenger = new FeedMessageOperatorDescriptor(jobSpec, dataverse, dataset,
                 feedMessages);
-        return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(feedMessenger, spPc.second);
+        return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(feedMessenger, partitionConstraint);
     }
 
     public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildBtreeRuntime(JobSpecification jobSpec,
@@ -1477,6 +1491,14 @@
         return type.getDatatype();
     }
 
+    public FeedPolicy findFeedPolicy(String dataverse, String policyName) throws AlgebricksException {
+        try {
+            return MetadataManager.INSTANCE.getFeedPolicy(mdTxnCtx, dataverse, policyName);
+        } catch (MetadataException e) {
+            throw new AlgebricksException(e);
+        }
+    }
+
     public List<Index> getDatasetIndexes(String dataverseName, String datasetName) throws AlgebricksException {
         try {
             return MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
@@ -1515,8 +1537,8 @@
     private Map<String, Object> wrapProperties(Map<String, String> properties) {
         Map<String, Object> wrappedProperties = new HashMap<String, Object>();
         wrappedProperties.putAll(properties);
-        wrappedProperties.put(HDFSAdapterFactory.SCHEDULER, hdfsScheduler);
-        wrappedProperties.put(HDFSAdapterFactory.CLUSTER_LOCATIONS, getClusterLocations());
+        wrappedProperties.put(SCHEDULER, hdfsScheduler);
+        wrappedProperties.put(CLUSTER_LOCATIONS, getClusterLocations());
         return wrappedProperties;
     }
 
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/FileSplitDataSink.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/FileSplitDataSink.java
index 3e61ec5..0827f09 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/FileSplitDataSink.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/FileSplitDataSink.java
@@ -18,6 +18,7 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
 
+
 public class FileSplitDataSink implements IDataSink {
 
     private FileSplitSinkId id;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/FileSplitSinkId.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/FileSplitSinkId.java
index 5801002..9ca7291 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/FileSplitSinkId.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/FileSplitSinkId.java
@@ -17,6 +17,7 @@
 
 import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
 
+
 public class FileSplitSinkId {
 
     private FileSplit fileSplit;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/ResultSetDataSink.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/ResultSetDataSink.java
index 3fd9285..50c6225 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/ResultSetDataSink.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/ResultSetDataSink.java
@@ -20,6 +20,7 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ResultSetDomain;
 
+
 public class ResultSetDataSink implements IDataSink {
 
     private ResultSetSinkId id;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/ResultSetSinkId.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/ResultSetSinkId.java
index 99d026b..4fa3907 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/ResultSetSinkId.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/ResultSetSinkId.java
@@ -16,6 +16,7 @@
 
 import edu.uci.ics.hyracks.api.dataset.ResultSetId;
 
+
 public class ResultSetSinkId {
 
     private final ResultSetId resultSetId;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/DatasourceAdapter.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/DatasourceAdapter.java
index 3c72084..5349654 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/DatasourceAdapter.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/DatasourceAdapter.java
@@ -14,9 +14,9 @@
  */
 package edu.uci.ics.asterix.metadata.entities;
 
-import edu.uci.ics.asterix.external.dataset.adapter.AdapterIdentifier;
 import edu.uci.ics.asterix.metadata.MetadataCache;
 import edu.uci.ics.asterix.metadata.api.IMetadataEntity;
+import edu.uci.ics.asterix.metadata.feeds.AdapterIdentifier;
 
 public class DatasourceAdapter implements IMetadataEntity {
 
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/FeedActivity.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/FeedActivity.java
new file mode 100644
index 0000000..72a1f03
--- /dev/null
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/FeedActivity.java
@@ -0,0 +1,149 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.metadata.entities;
+
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.asterix.metadata.MetadataCache;
+import edu.uci.ics.asterix.metadata.api.IMetadataEntity;
+
+/**
+ * Metadata describing a feed activity record.
+ */
+public class FeedActivity implements IMetadataEntity, Comparable<FeedActivity> {
+
+    private static final long serialVersionUID = 1L;
+
+    private int activityId;
+
+    private final String dataverseName;
+    // Enforced to be unique within a dataverse.
+    private final String datasetName;
+
+    private String lastUpdatedTimestamp;
+    private FeedActivityType activityType;
+    private Map<String, String> feedActivityDetails;
+
+    public static enum FeedActivityType {
+        FEED_BEGIN,
+        FEED_END,
+        FEED_FAILURE,
+        FEED_STATS,
+        FEED_EXPAND,
+        FEED_SHRINK
+    }
+
+    public static class FeedActivityDetails {
+        public static final String COMPUTE_LOCATIONS = "compute-locations";
+        public static final String INGEST_LOCATIONS = "ingest-locations";
+        public static final String TOTAL_INGESTED = "total-ingested";
+        public static final String INGESTION_RATE = "ingestion-rate";
+        public static final String EXCEPTION_LOCATION = "exception-location";
+        public static final String EXCEPTION_MESSAGE = "exception-message";
+
+    }
+
+    public FeedActivity(String dataverseName, String datasetName, FeedActivityType feedActivityType,
+            Map<String, String> feedActivityDetails) {
+        this.dataverseName = dataverseName;
+        this.datasetName = datasetName;
+        this.activityType = feedActivityType;
+        this.feedActivityDetails = feedActivityDetails;
+    }
+
+    public String getDataverseName() {
+        return dataverseName;
+    }
+
+    public String getDatasetName() {
+        return datasetName;
+    }
+
+    @Override
+    public Object addToCache(MetadataCache cache) {
+        return cache.addFeedActivityIfNotExists(this);
+    }
+
+    @Override
+    public Object dropFromCache(MetadataCache cache) {
+        return cache.dropFeedActivity(this);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+        if (!(other instanceof FeedActivity)) {
+            return false;
+        }
+        FeedActivity otherDataset = (FeedActivity) other;
+        if (!otherDataset.dataverseName.equals(dataverseName)) {
+            return false;
+        }
+        if (!otherDataset.datasetName.equals(datasetName)) {
+            return false;
+        }
+        return true;
+    }
+
+    public FeedActivityType getFeedActivityType() {
+        return activityType;
+    }
+
+    public void setFeedActivityType(FeedActivityType feedActivityType) {
+        this.activityType = feedActivityType;
+    }
+
+    public String getLastUpdatedTimestamp() {
+        return lastUpdatedTimestamp;
+    }
+
+    public void setLastUpdatedTimestamp(String lastUpdatedTimestamp) {
+        this.lastUpdatedTimestamp = lastUpdatedTimestamp;
+    }
+
+    public int getActivityId() {
+        return activityId;
+    }
+
+    public void setActivityId(int activityId) {
+        this.activityId = activityId;
+    }
+
+    public Map<String, String> getFeedActivityDetails() {
+        return feedActivityDetails;
+    }
+
+    public void setFeedActivityDetails(Map<String, String> feedActivityDetails) {
+        this.feedActivityDetails = feedActivityDetails;
+    }
+
+    public FeedActivityType getActivityType() {
+        return activityType;
+    }
+
+    public void setActivityType(FeedActivityType activityType) {
+        this.activityType = activityType;
+    }
+
+    @Override
+    public int compareTo(FeedActivity o) {
+        return this.activityId - o.getActivityId();
+    }
+
+}
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/FeedDatasetDetails.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/FeedDatasetDetails.java
index 22de3d3..81be3b5 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/FeedDatasetDetails.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/FeedDatasetDetails.java
@@ -23,6 +23,7 @@
 import edu.uci.ics.asterix.builders.IARecordBuilder;
 import edu.uci.ics.asterix.builders.OrderedListBuilder;
 import edu.uci.ics.asterix.builders.RecordBuilder;
+import edu.uci.ics.asterix.builders.UnorderedListBuilder;
 import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
 import edu.uci.ics.asterix.common.functions.FunctionSignature;
@@ -31,6 +32,7 @@
 import edu.uci.ics.asterix.om.base.AMutableString;
 import edu.uci.ics.asterix.om.base.AString;
 import edu.uci.ics.asterix.om.types.AOrderedListType;
+import edu.uci.ics.asterix.om.types.AUnorderedListType;
 import edu.uci.ics.asterix.om.types.BuiltinType;
 import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
@@ -48,25 +50,31 @@
     private final Map<String, String> properties;
     private final FunctionSignature signature;
     private FeedState feedState;
+    private List<String> ingestNodes;
+    private List<String> computeNodes;
 
     public enum FeedState {
-        INACTIVE,
         // INACTIVE state signifies that the feed dataset is not
         // connected with the external world through the feed
         // adapter.
-        ACTIVE
+        INACTIVE,
+
         // ACTIVE state signifies that the feed dataset is connected to the
         // external world using an adapter that may put data into the dataset.
+        ACTIVE
     }
 
     public FeedDatasetDetails(FileStructure fileStructure, PartitioningStrategy partitioningStrategy,
             List<String> partitioningKey, List<String> primaryKey, String groupName, String adapterFactory,
-            Map<String, String> properties, FunctionSignature signature, String feedState) {
+            Map<String, String> properties, FunctionSignature signature, String feedState, List<String> ingestNodes,
+            List<String> computeNodes) {
         super(fileStructure, partitioningStrategy, partitioningKey, primaryKey, groupName);
         this.properties = properties;
         this.adapterFactory = adapterFactory;
         this.signature = signature;
         this.feedState = feedState.equals(FeedState.ACTIVE.toString()) ? FeedState.ACTIVE : FeedState.INACTIVE;
+        this.ingestNodes = ingestNodes;
+        this.computeNodes = computeNodes;
     }
 
     @Override
@@ -160,6 +168,38 @@
         stringSerde.serialize(aString, fieldValue.getDataOutput());
         feedRecordBuilder.addField(MetadataRecordTypes.FEED_DETAILS_ARECORD_STATE_FIELD_INDEX, fieldValue);
 
+        // write field 9
+        UnorderedListBuilder unorderedlistBuilder = new UnorderedListBuilder();
+        unorderedlistBuilder
+                .reset((AUnorderedListType) MetadataRecordTypes.FEED_DETAILS_RECORDTYPE.getFieldTypes()[MetadataRecordTypes.FEED_DETAILS_ARECORD_INGEST_NODES_FIELD_INDEX]);
+        itemValue = new ArrayBackedValueStorage();
+        if (ingestNodes != null) {
+            for (String node : ingestNodes) {
+                itemValue.reset();
+                aString.setValue(node);
+                listBuilder.addItem(itemValue);
+            }
+        }
+        fieldValue.reset();
+        unorderedlistBuilder.write(fieldValue.getDataOutput(), true);
+        feedRecordBuilder.addField(MetadataRecordTypes.FEED_DETAILS_ARECORD_INGEST_NODES_FIELD_INDEX, fieldValue);
+
+        // write field 10
+        unorderedlistBuilder = new UnorderedListBuilder();
+        unorderedlistBuilder
+                .reset((AUnorderedListType) MetadataRecordTypes.FEED_DETAILS_RECORDTYPE.getFieldTypes()[MetadataRecordTypes.FEED_DETAILS_ARECORD_COMPUTE_NODES_FIELD_INDEX]);
+        itemValue = new ArrayBackedValueStorage();
+        if (computeNodes != null) {
+            for (String node : computeNodes) {
+                itemValue.reset();
+                aString.setValue(node);
+                listBuilder.addItem(itemValue);
+            }
+        }
+        fieldValue.reset();
+        unorderedlistBuilder.write(fieldValue.getDataOutput(), true);
+        feedRecordBuilder.addField(MetadataRecordTypes.FEED_DETAILS_ARECORD_COMPUTE_NODES_FIELD_INDEX, fieldValue);
+
         try {
             feedRecordBuilder.write(out, true);
         } catch (IOException | AsterixException e) {
@@ -215,4 +255,20 @@
         return signature;
     }
 
+    public List<String> getIngestNodes() {
+        return ingestNodes;
+    }
+
+    public void setIngestNodes(List<String> ingestNodes) {
+        this.ingestNodes = ingestNodes;
+    }
+
+    public List<String> getComputeNodes() {
+        return computeNodes;
+    }
+
+    public void setComputeNodes(List<String> computeNodes) {
+        this.computeNodes = computeNodes;
+    }
+
 }
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/FeedPolicy.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/FeedPolicy.java
new file mode 100644
index 0000000..b011e5c
--- /dev/null
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/FeedPolicy.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.metadata.entities;
+
+import java.util.Map;
+
+import edu.uci.ics.asterix.metadata.MetadataCache;
+import edu.uci.ics.asterix.metadata.api.IMetadataEntity;
+
+/**
+ * Metadata describing a feed activity record.
+ */
+public class FeedPolicy implements IMetadataEntity {
+
+    private static final long serialVersionUID = 1L;
+
+    private final String dataverseName;
+    // Enforced to be unique within a dataverse.
+    private final String policyName;
+    // A description of the policy
+    private final String description;
+    // The policy properties associated with the feed dataset
+    private Map<String, String> properties;
+
+    public FeedPolicy(String dataverseName, String policyName, String description, Map<String, String> properties) {
+        this.dataverseName = dataverseName;
+        this.policyName = policyName;
+        this.description = description;
+        this.properties = properties;
+    }
+
+    public String getDataverseName() {
+        return dataverseName;
+    }
+
+    public String getPolicyName() {
+        return policyName;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+        if (!(other instanceof FeedPolicy)) {
+            return false;
+        }
+        FeedPolicy otherPolicy = (FeedPolicy) other;
+        if (!otherPolicy.dataverseName.equals(dataverseName)) {
+            return false;
+        }
+        if (!otherPolicy.policyName.equals(policyName)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public Object addToCache(MetadataCache cache) {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Object dropFromCache(MetadataCache cache) {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public Map<String, String> getProperties() {
+        return properties;
+    }
+
+    public void setProperties(Map<String, String> properties) {
+        this.properties = properties;
+    }
+
+}
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/AbstractTupleTranslator.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/AbstractTupleTranslator.java
index dc8c390..a85787f 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/AbstractTupleTranslator.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/AbstractTupleTranslator.java
@@ -20,6 +20,7 @@
 import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
 import edu.uci.ics.asterix.metadata.api.IMetadataEntityTupleTranslator;
 import edu.uci.ics.asterix.om.base.ABoolean;
+import edu.uci.ics.asterix.om.base.AInt32;
 import edu.uci.ics.asterix.om.base.AMutableString;
 import edu.uci.ics.asterix.om.base.AString;
 import edu.uci.ics.asterix.om.types.BuiltinType;
@@ -40,6 +41,10 @@
     @SuppressWarnings("unchecked")
     protected ISerializerDeserializer<ABoolean> booleanSerde = AqlSerializerDeserializerProvider.INSTANCE
             .getSerializerDeserializer(BuiltinType.ABOOLEAN);
+    @SuppressWarnings("unchecked")
+    protected ISerializerDeserializer<AInt32> int32Serde = AqlSerializerDeserializerProvider.INSTANCE
+            .getSerializerDeserializer(BuiltinType.AINT32);
+
     protected final IARecordBuilder recordBuilder;
     protected final ArrayBackedValueStorage fieldValue;
     protected final ArrayTupleBuilder tupleBuilder;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
index 61f856a..65b2d78 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
@@ -138,9 +138,9 @@
                 String value;
                 while (cursor.next()) {
                     ARecord field = (ARecord) cursor.get();
-                    key = ((AString) field.getValueByPos(MetadataRecordTypes.DATASOURCE_PROPERTIES_NAME_FIELD_INDEX))
+                    key = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_NAME_FIELD_INDEX))
                             .getStringValue();
-                    value = ((AString) field.getValueByPos(MetadataRecordTypes.DATASOURCE_PROPERTIES_VALUE_FIELD_INDEX))
+                    value = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_VALUE_FIELD_INDEX))
                             .getStringValue();
                     properties.put(key, value);
                 }
@@ -169,8 +169,23 @@
                 String feedState = ((AString) datasetDetailsRecord
                         .getValueByPos(MetadataRecordTypes.FEED_DETAILS_ARECORD_STATE_FIELD_INDEX)).getStringValue();
 
+                cursor = ((AUnorderedList) datasetDetailsRecord
+                        .getValueByPos(MetadataRecordTypes.FEED_DETAILS_ARECORD_INGEST_NODES_FIELD_INDEX)).getCursor();
+                List<String> ingestNodes = new ArrayList<String>();
+                while (cursor.next()) {
+                    ingestNodes.add(((AString) cursor.get()).getStringValue());
+                }
+
+                cursor = ((AUnorderedList) datasetDetailsRecord
+                        .getValueByPos(MetadataRecordTypes.FEED_DETAILS_ARECORD_INGEST_NODES_FIELD_INDEX)).getCursor();
+                List<String> computeNodes = new ArrayList<String>();
+                while (cursor.next()) {
+                    computeNodes.add(((AString) cursor.get()).getStringValue());
+                }
+
                 datasetDetails = new FeedDatasetDetails(fileStructure, partitioningStrategy, partitioningKey,
-                        partitioningKey, groupName, adapter, properties, signature, feedState);
+                        partitioningKey, groupName, adapter, properties, signature, feedState, ingestNodes,
+                        computeNodes);
                 break;
             }
             case INTERNAL: {
@@ -213,18 +228,19 @@
                 String value;
                 while (cursor.next()) {
                     ARecord field = (ARecord) cursor.get();
-                    key = ((AString) field.getValueByPos(MetadataRecordTypes.DATASOURCE_PROPERTIES_NAME_FIELD_INDEX))
+                    key = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_NAME_FIELD_INDEX))
                             .getStringValue();
-                    value = ((AString) field.getValueByPos(MetadataRecordTypes.DATASOURCE_PROPERTIES_VALUE_FIELD_INDEX))
+                    value = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_VALUE_FIELD_INDEX))
                             .getStringValue();
                     properties.put(key, value);
                 }
                 datasetDetails = new ExternalDatasetDetails(adapter, properties);
         }
-        
+
         Map<String, String> hints = getDatasetHints(datasetRecord);
-        
-        return new Dataset(dataverseName, datasetName, typeName, datasetDetails, hints, datasetType, datasetId, pendingOp);
+
+        return new Dataset(dataverseName, datasetName, typeName, datasetDetails, hints, datasetType, datasetId,
+                pendingOp);
     }
 
     @Override
@@ -343,9 +359,9 @@
         IACursor cursor = list.getCursor();
         while (cursor.next()) {
             ARecord field = (ARecord) cursor.get();
-            key = ((AString) field.getValueByPos(MetadataRecordTypes.DATASOURCE_PROPERTIES_NAME_FIELD_INDEX))
+            key = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_NAME_FIELD_INDEX))
                     .getStringValue();
-            value = ((AString) field.getValueByPos(MetadataRecordTypes.DATASOURCE_PROPERTIES_VALUE_FIELD_INDEX))
+            value = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_VALUE_FIELD_INDEX))
                     .getStringValue();
             hints.put(key, value);
         }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java
index 4a5e4dcf..e93fd97 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java
@@ -22,13 +22,13 @@
 import java.util.Calendar;
 
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
-import edu.uci.ics.asterix.external.dataset.adapter.AdapterIdentifier;
 import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
 import edu.uci.ics.asterix.metadata.MetadataException;
 import edu.uci.ics.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
 import edu.uci.ics.asterix.metadata.bootstrap.MetadataRecordTypes;
 import edu.uci.ics.asterix.metadata.entities.DatasourceAdapter;
 import edu.uci.ics.asterix.metadata.entities.DatasourceAdapter.AdapterType;
+import edu.uci.ics.asterix.metadata.feeds.AdapterIdentifier;
 import edu.uci.ics.asterix.om.base.ARecord;
 import edu.uci.ics.asterix.om.base.AString;
 import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/FeedActivityTupleTranslator.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/FeedActivityTupleTranslator.java
new file mode 100644
index 0000000..51678dc2
--- /dev/null
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/FeedActivityTupleTranslator.java
@@ -0,0 +1,227 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.metadata.entitytupletranslators;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.asterix.builders.IARecordBuilder;
+import edu.uci.ics.asterix.builders.RecordBuilder;
+import edu.uci.ics.asterix.builders.UnorderedListBuilder;
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import edu.uci.ics.asterix.metadata.MetadataException;
+import edu.uci.ics.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
+import edu.uci.ics.asterix.metadata.bootstrap.MetadataRecordTypes;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity.FeedActivityType;
+import edu.uci.ics.asterix.om.base.AInt32;
+import edu.uci.ics.asterix.om.base.AMutableInt32;
+import edu.uci.ics.asterix.om.base.AMutableString;
+import edu.uci.ics.asterix.om.base.ARecord;
+import edu.uci.ics.asterix.om.base.AString;
+import edu.uci.ics.asterix.om.base.AUnorderedList;
+import edu.uci.ics.asterix.om.base.IACursor;
+import edu.uci.ics.asterix.om.types.AUnorderedListType;
+import edu.uci.ics.asterix.om.types.BuiltinType;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+/**
+ * Translates a Dataset metadata entity to an ITupleReference and vice versa.
+ */
+public class FeedActivityTupleTranslator extends AbstractTupleTranslator<FeedActivity> {
+    // Field indexes of serialized FeedActivity in a tuple.
+    // Key field.
+    public static final int FEED_ACTIVITY_ACTIVITY_DATAVERSE_NAME_FIELD_INDEX = 0;
+
+    public static final int FEED_ACTIVITY_ACTIVITY_DATASET_NAME_FIELD_INDEX = 1;
+
+    public static final int FEED_ACTIVITY_ACTIVITY_ID_FIELD_INDEX = 2;
+
+    // Payload field containing serialized FeedActivity.
+    public static final int FEED_ACTIVITY_PAYLOAD_TUPLE_FIELD_INDEX = 3;
+
+    @SuppressWarnings("unchecked")
+    private ISerializerDeserializer<ARecord> recordSerDes = AqlSerializerDeserializerProvider.INSTANCE
+            .getSerializerDeserializer(MetadataRecordTypes.FEED_ACTIVITY_RECORDTYPE);
+    private AMutableInt32 aInt32;
+    protected ISerializerDeserializer<AInt32> aInt32Serde;
+
+    @SuppressWarnings("unchecked")
+    public FeedActivityTupleTranslator(boolean getTuple) {
+        super(getTuple, MetadataPrimaryIndexes.FEED_ACTIVITY_DATASET.getFieldCount());
+        aInt32 = new AMutableInt32(-1);
+        aInt32Serde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT32);
+    }
+
+    @Override
+    public FeedActivity getMetadataEntytiFromTuple(ITupleReference frameTuple) throws IOException {
+        byte[] serRecord = frameTuple.getFieldData(FEED_ACTIVITY_PAYLOAD_TUPLE_FIELD_INDEX);
+        int recordStartOffset = frameTuple.getFieldStart(FEED_ACTIVITY_PAYLOAD_TUPLE_FIELD_INDEX);
+        int recordLength = frameTuple.getFieldLength(FEED_ACTIVITY_PAYLOAD_TUPLE_FIELD_INDEX);
+        ByteArrayInputStream stream = new ByteArrayInputStream(serRecord, recordStartOffset, recordLength);
+        DataInput in = new DataInputStream(stream);
+        ARecord feedActivityRecord = (ARecord) recordSerDes.deserialize(in);
+        return createFeedActivityFromARecord(feedActivityRecord);
+    }
+
+    private FeedActivity createFeedActivityFromARecord(ARecord feedActivityRecord) {
+
+        String dataverseName = ((AString) feedActivityRecord
+                .getValueByPos(MetadataRecordTypes.FEED_ACTIVITY_ARECORD_DATAVERSE_NAME_FIELD_INDEX)).getStringValue();
+        String datasetName = ((AString) feedActivityRecord
+                .getValueByPos(MetadataRecordTypes.FEED_ACTIVITY_ARECORD_DATASET_NAME_FIELD_INDEX)).getStringValue();
+        int activityId = ((AInt32) feedActivityRecord
+                .getValueByPos(MetadataRecordTypes.FEED_ACTIVITY_ARECORD_ACTIVITY_ID_FIELD_INDEX)).getIntegerValue();
+        String feedActivityType = ((AString) feedActivityRecord
+                .getValueByPos(MetadataRecordTypes.FEED_ACTIVITY_ARECORD_ACTIVITY_TYPE_FIELD_INDEX)).getStringValue();
+
+        IACursor cursor = ((AUnorderedList) feedActivityRecord
+                .getValueByPos(MetadataRecordTypes.FEED_ACTIVITY_ARECORD_DETAILS_FIELD_INDEX)).getCursor();
+        Map<String, String> activityDetails = new HashMap<String, String>();
+        String key;
+        String value;
+        while (cursor.next()) {
+            ARecord field = (ARecord) cursor.get();
+            key = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_NAME_FIELD_INDEX)).getStringValue();
+            value = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_VALUE_FIELD_INDEX)).getStringValue();
+            activityDetails.put(key, value);
+        }
+
+        FeedActivity fa = new FeedActivity(dataverseName, datasetName, FeedActivityType.valueOf(feedActivityType),
+                activityDetails);
+        fa.setActivityId(activityId);
+        return fa;
+    }
+
+    @Override
+    public ITupleReference getTupleFromMetadataEntity(FeedActivity feedActivity) throws IOException, MetadataException {
+        // write the key in the first three fields of the tuple
+        ArrayBackedValueStorage itemValue = new ArrayBackedValueStorage();
+
+        tupleBuilder.reset();
+        aString.setValue(feedActivity.getDataverseName());
+        stringSerde.serialize(aString, tupleBuilder.getDataOutput());
+        tupleBuilder.addFieldEndOffset();
+
+        aString.setValue(feedActivity.getDatasetName());
+        stringSerde.serialize(aString, tupleBuilder.getDataOutput());
+        tupleBuilder.addFieldEndOffset();
+
+        aInt32.setValue(feedActivity.getActivityId());
+        int32Serde.serialize(aInt32, tupleBuilder.getDataOutput());
+        tupleBuilder.addFieldEndOffset();
+        // write the pay-load in the 2nd field of the tuple
+
+        recordBuilder.reset(MetadataRecordTypes.FEED_ACTIVITY_RECORDTYPE);
+
+        // write field 0
+        fieldValue.reset();
+        aString.setValue(feedActivity.getDataverseName());
+        stringSerde.serialize(aString, fieldValue.getDataOutput());
+        recordBuilder.addField(MetadataRecordTypes.FEED_ACTIVITY_ARECORD_DATAVERSE_NAME_FIELD_INDEX, fieldValue);
+
+        // write field 1
+        fieldValue.reset();
+        aString.setValue(feedActivity.getDatasetName());
+        stringSerde.serialize(aString, fieldValue.getDataOutput());
+        recordBuilder.addField(MetadataRecordTypes.FEED_ACTIVITY_ARECORD_DATASET_NAME_FIELD_INDEX, fieldValue);
+
+        // write field 2
+        fieldValue.reset();
+        aInt32.setValue(feedActivity.getActivityId());
+        int32Serde.serialize(aInt32, fieldValue.getDataOutput());
+        recordBuilder.addField(MetadataRecordTypes.FEED_ACTIVITY_ARECORD_ACTIVITY_ID_FIELD_INDEX, fieldValue);
+
+        // write field 3
+        fieldValue.reset();
+        aString.setValue(feedActivity.getFeedActivityType().name());
+        stringSerde.serialize(aString, fieldValue.getDataOutput());
+        recordBuilder.addField(MetadataRecordTypes.FEED_ACTIVITY_ARECORD_ACTIVITY_TYPE_FIELD_INDEX, fieldValue);
+
+        // write field 4
+        fieldValue.reset();
+        aString.setValue(Calendar.getInstance().getTime().toString());
+        stringSerde.serialize(aString, fieldValue.getDataOutput());
+        recordBuilder.addField(MetadataRecordTypes.FEED_ACTIVITY_ARECORD_LAST_UPDATE_TIMESTAMP_FIELD_INDEX, fieldValue);
+
+        // write field 5
+        Map<String, String> properties = feedActivity.getFeedActivityDetails();
+        UnorderedListBuilder listBuilder = new UnorderedListBuilder();
+        listBuilder
+                .reset((AUnorderedListType) MetadataRecordTypes.FEED_ACTIVITY_RECORDTYPE.getFieldTypes()[MetadataRecordTypes.FEED_ACTIVITY_ARECORD_DETAILS_FIELD_INDEX]);
+        for (Map.Entry<String, String> property : properties.entrySet()) {
+            String name = property.getKey();
+            String value = property.getValue();
+            itemValue.reset();
+            writePropertyTypeRecord(name, value, itemValue.getDataOutput());
+            listBuilder.addItem(itemValue);
+        }
+        fieldValue.reset();
+        listBuilder.write(fieldValue.getDataOutput(), true);
+        recordBuilder.addField(MetadataRecordTypes.FEED_ACTIVITY_ARECORD_DETAILS_FIELD_INDEX, fieldValue);
+
+        // write record
+        try {
+            recordBuilder.write(tupleBuilder.getDataOutput(), true);
+        } catch (AsterixException e) {
+            throw new MetadataException(e);
+        }
+        tupleBuilder.addFieldEndOffset();
+
+        tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
+        return tuple;
+    }
+
+    public void writePropertyTypeRecord(String name, String value, DataOutput out) throws HyracksDataException {
+        IARecordBuilder propertyRecordBuilder = new RecordBuilder();
+        ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
+        propertyRecordBuilder.reset(MetadataRecordTypes.FEED_ACTIVITY_DETAILS_RECORDTYPE);
+        AMutableString aString = new AMutableString("");
+        ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
+                .getSerializerDeserializer(BuiltinType.ASTRING);
+
+        // write field 0
+        fieldValue.reset();
+        aString.setValue(name);
+        stringSerde.serialize(aString, fieldValue.getDataOutput());
+        propertyRecordBuilder.addField(0, fieldValue);
+
+        // write field 1
+        fieldValue.reset();
+        aString.setValue(value);
+        stringSerde.serialize(aString, fieldValue.getDataOutput());
+        propertyRecordBuilder.addField(1, fieldValue);
+
+        try {
+            propertyRecordBuilder.write(out, true);
+        } catch (IOException | AsterixException e) {
+            throw new HyracksDataException(e);
+        }
+    }
+
+}
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java
new file mode 100644
index 0000000..57d2d1c
--- /dev/null
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java
@@ -0,0 +1,200 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.metadata.entitytupletranslators;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import edu.uci.ics.asterix.builders.IARecordBuilder;
+import edu.uci.ics.asterix.builders.RecordBuilder;
+import edu.uci.ics.asterix.builders.UnorderedListBuilder;
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import edu.uci.ics.asterix.metadata.MetadataException;
+import edu.uci.ics.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
+import edu.uci.ics.asterix.metadata.bootstrap.MetadataRecordTypes;
+import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
+import edu.uci.ics.asterix.om.base.AInt32;
+import edu.uci.ics.asterix.om.base.AMutableInt32;
+import edu.uci.ics.asterix.om.base.AMutableString;
+import edu.uci.ics.asterix.om.base.ARecord;
+import edu.uci.ics.asterix.om.base.AString;
+import edu.uci.ics.asterix.om.base.AUnorderedList;
+import edu.uci.ics.asterix.om.base.IACursor;
+import edu.uci.ics.asterix.om.types.AUnorderedListType;
+import edu.uci.ics.asterix.om.types.BuiltinType;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+/**
+ * Translates a Dataset metadata entity to an ITupleReference and vice versa.
+ */
+public class FeedPolicyTupleTranslator extends AbstractTupleTranslator<FeedPolicy> {
+    // Field indexes of serialized FeedPolicy in a tuple.
+    // Key field.
+    public static final int FEED_POLICY_DATAVERSE_NAME_FIELD_INDEX = 0;
+
+    public static final int FEED_POLICY_POLICY_NAME_FIELD_INDEX = 1;
+
+    // Payload field containing serialized feedPolicy.
+    public static final int FEED_POLICY_PAYLOAD_TUPLE_FIELD_INDEX = 2;
+
+    @SuppressWarnings("unchecked")
+    private ISerializerDeserializer<ARecord> recordSerDes = AqlSerializerDeserializerProvider.INSTANCE
+            .getSerializerDeserializer(MetadataRecordTypes.FEED_POLICY_RECORDTYPE);
+    private AMutableInt32 aInt32;
+    protected ISerializerDeserializer<AInt32> aInt32Serde;
+
+    @SuppressWarnings("unchecked")
+    public FeedPolicyTupleTranslator(boolean getTuple) {
+        super(getTuple, MetadataPrimaryIndexes.FEED_POLICY_DATASET.getFieldCount());
+        aInt32 = new AMutableInt32(-1);
+        aInt32Serde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT32);
+    }
+
+    @Override
+    public FeedPolicy getMetadataEntytiFromTuple(ITupleReference frameTuple) throws IOException {
+        byte[] serRecord = frameTuple.getFieldData(FEED_POLICY_PAYLOAD_TUPLE_FIELD_INDEX);
+        int recordStartOffset = frameTuple.getFieldStart(FEED_POLICY_PAYLOAD_TUPLE_FIELD_INDEX);
+        int recordLength = frameTuple.getFieldLength(FEED_POLICY_PAYLOAD_TUPLE_FIELD_INDEX);
+        ByteArrayInputStream stream = new ByteArrayInputStream(serRecord, recordStartOffset, recordLength);
+        DataInput in = new DataInputStream(stream);
+        ARecord feedPolicyRecord = (ARecord) recordSerDes.deserialize(in);
+        return createFeedPolicyFromARecord(feedPolicyRecord);
+    }
+
+    private FeedPolicy createFeedPolicyFromARecord(ARecord feedPolicyRecord) {
+        FeedPolicy feedPolicy = null;
+        String dataverseName = ((AString) feedPolicyRecord
+                .getValueByPos(MetadataRecordTypes.FEED_POLICY_ARECORD_DATAVERSE_NAME_FIELD_INDEX)).getStringValue();
+        String policyName = ((AString) feedPolicyRecord
+                .getValueByPos(MetadataRecordTypes.FEED_POLICY_ARECORD_POLICY_NAME_FIELD_INDEX)).getStringValue();
+
+        String description = ((AString) feedPolicyRecord
+                .getValueByPos(MetadataRecordTypes.FEED_POLICY_ARECORD_DESCRIPTION_FIELD_INDEX)).getStringValue();
+
+        IACursor cursor = ((AUnorderedList) feedPolicyRecord
+                .getValueByPos(MetadataRecordTypes.FEED_POLICY_ARECORD_PROPERTIES_FIELD_INDEX)).getCursor();
+        Map<String, String> policyParamters = new HashMap<String, String>();
+        String key;
+        String value;
+        while (cursor.next()) {
+            ARecord field = (ARecord) cursor.get();
+            key = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_NAME_FIELD_INDEX)).getStringValue();
+            value = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_VALUE_FIELD_INDEX)).getStringValue();
+            policyParamters.put(key, value);
+        }
+
+        feedPolicy = new FeedPolicy(dataverseName, policyName, description, policyParamters);
+        return feedPolicy;
+    }
+
+    @Override
+    public ITupleReference getTupleFromMetadataEntity(FeedPolicy feedPolicy) throws IOException, MetadataException {
+        // write the key in the first three fields of the tuple
+        ArrayBackedValueStorage itemValue = new ArrayBackedValueStorage();
+
+        tupleBuilder.reset();
+        aString.setValue(feedPolicy.getDataverseName());
+        stringSerde.serialize(aString, tupleBuilder.getDataOutput());
+        tupleBuilder.addFieldEndOffset();
+
+        aString.setValue(feedPolicy.getPolicyName());
+        stringSerde.serialize(aString, tupleBuilder.getDataOutput());
+        tupleBuilder.addFieldEndOffset();
+
+        recordBuilder.reset(MetadataRecordTypes.FEED_POLICY_RECORDTYPE);
+
+        // write field 0
+        fieldValue.reset();
+        aString.setValue(feedPolicy.getDataverseName());
+        stringSerde.serialize(aString, fieldValue.getDataOutput());
+        recordBuilder.addField(MetadataRecordTypes.FEED_POLICY_ARECORD_DATAVERSE_NAME_FIELD_INDEX, fieldValue);
+
+        // write field 1
+        fieldValue.reset();
+        aString.setValue(feedPolicy.getPolicyName());
+        stringSerde.serialize(aString, fieldValue.getDataOutput());
+        recordBuilder.addField(MetadataRecordTypes.FEED_POLICY_ARECORD_POLICY_NAME_FIELD_INDEX, fieldValue);
+
+        // write field 2
+        fieldValue.reset();
+        aString.setValue(feedPolicy.getDescription());
+        stringSerde.serialize(aString, fieldValue.getDataOutput());
+        recordBuilder.addField(MetadataRecordTypes.FEED_POLICY_ARECORD_POLICY_NAME_FIELD_INDEX, fieldValue);
+
+        // write field 3 (properties)
+        Map<String, String> properties = feedPolicy.getProperties();
+        UnorderedListBuilder listBuilder = new UnorderedListBuilder();
+        listBuilder
+                .reset((AUnorderedListType) MetadataRecordTypes.FEED_POLICY_RECORDTYPE.getFieldTypes()[MetadataRecordTypes.FEED_POLICY_ARECORD_PROPERTIES_FIELD_INDEX]);
+        for (Map.Entry<String, String> property : properties.entrySet()) {
+            String name = property.getKey();
+            String value = property.getValue();
+            itemValue.reset();
+            writePropertyTypeRecord(name, value, itemValue.getDataOutput());
+            listBuilder.addItem(itemValue);
+        }
+        fieldValue.reset();
+        listBuilder.write(fieldValue.getDataOutput(), true);
+        recordBuilder.addField(MetadataRecordTypes.FEED_POLICY_ARECORD_PROPERTIES_FIELD_INDEX, fieldValue);
+
+        // write record
+        try {
+            recordBuilder.write(tupleBuilder.getDataOutput(), true);
+        } catch (AsterixException e) {
+            throw new MetadataException(e);
+        }
+        tupleBuilder.addFieldEndOffset();
+
+        tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
+        return tuple;
+    }
+
+    public void writePropertyTypeRecord(String name, String value, DataOutput out) throws HyracksDataException {
+        IARecordBuilder propertyRecordBuilder = new RecordBuilder();
+        ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
+        propertyRecordBuilder.reset(MetadataRecordTypes.POLICY_PARAMS_RECORDTYPE);
+        AMutableString aString = new AMutableString("");
+        ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
+                .getSerializerDeserializer(BuiltinType.ASTRING);
+
+        // write field 0
+        fieldValue.reset();
+        aString.setValue(name);
+        stringSerde.serialize(aString, fieldValue.getDataOutput());
+        propertyRecordBuilder.addField(0, fieldValue);
+
+        // write field 1
+        fieldValue.reset();
+        aString.setValue(value);
+        stringSerde.serialize(aString, fieldValue.getDataOutput());
+        propertyRecordBuilder.addField(1, fieldValue);
+
+        try {
+            propertyRecordBuilder.write(out, true);
+        } catch (IOException | AsterixException e) {
+            throw new HyracksDataException(e);
+        }
+    }
+}
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/AbstractDatasourceAdapter.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/AbstractDatasourceAdapter.java
similarity index 96%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/AbstractDatasourceAdapter.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/AbstractDatasourceAdapter.java
index f96a030..7684969 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/AbstractDatasourceAdapter.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/AbstractDatasourceAdapter.java
@@ -12,11 +12,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.dataset.adapter;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.util.HashMap;
 import java.util.Map;
 
+import edu.uci.ics.asterix.metadata.feeds.IDatasourceAdapter;
 import edu.uci.ics.asterix.om.types.ATypeTag;
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/AbstractFeedDatasourceAdapter.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/AbstractFeedDatasourceAdapter.java
new file mode 100644
index 0000000..08d6972
--- /dev/null
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/AbstractFeedDatasourceAdapter.java
@@ -0,0 +1,106 @@
+package edu.uci.ics.asterix.metadata.feeds;
+
+import java.rmi.RemoteException;
+import java.util.HashMap;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import edu.uci.ics.asterix.common.exceptions.ACIDException;
+import edu.uci.ics.asterix.metadata.MetadataManager;
+import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity.FeedActivityDetails;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity.FeedActivityType;
+
+public abstract class AbstractFeedDatasourceAdapter extends AbstractDatasourceAdapter {
+
+    private static final long serialVersionUID = 1L;
+
+    private int batchPersistSize = 10;
+    protected FeedPolicyEnforcer policyEnforcer;
+    protected StatisticsCollector collector;
+    protected Timer timer;
+
+    public FeedPolicyEnforcer getPolicyEnforcer() {
+        return policyEnforcer;
+    }
+
+    public void setFeedPolicyEnforcer(FeedPolicyEnforcer policyEnforcer) {
+        this.policyEnforcer = policyEnforcer;
+        FeedPolicyAccessor policyAccessor = this.policyEnforcer.getFeedPolicyAccessor();
+        if (policyAccessor.collectStatistics()) {
+            long period = policyAccessor.getStatisicsCollectionPeriodInSecs();
+            collector = new StatisticsCollector(this, policyEnforcer.getFeedId(), period, batchPersistSize);
+            timer = new Timer();
+            timer.schedule(collector, period * 1000, period * 1000);
+        }
+    }
+
+    public abstract long getIngestedRecordsCount();
+
+    protected void dumpStatistics() throws RemoteException, ACIDException {
+        collector.persistStatistics(true);
+    }
+
+    private static class StatisticsCollector extends TimerTask {
+
+        private final AbstractFeedDatasourceAdapter adapter;
+        private final FeedId feedId;
+        //   private List<Long> previousCountValues = new ArrayList<Long>();
+        //   private List<Integer> ingestionRates = new ArrayList<Integer>();
+        private final long period;
+        private FeedActivity feedActivity;
+        private int numMeasurements = 0;
+        private int batchPersistSize;
+        private StringBuilder count = new StringBuilder();
+        private StringBuilder rate = new StringBuilder();
+        private long previousCount = 0;
+
+        public StatisticsCollector(AbstractFeedDatasourceAdapter adapter, FeedId feedId, long period,
+                int batchPersistSize) {
+            this.adapter = adapter;
+            this.feedId = feedId;
+            this.period = period;
+            this.batchPersistSize = batchPersistSize;
+            this.feedActivity = new FeedActivity(feedId.getDataverse(), feedId.getDataset(),
+                    FeedActivityType.FEED_STATS, new HashMap<String, String>());
+        }
+
+        @Override
+        public void run() {
+            try {
+                persistStatistics(false);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+
+        }
+
+        public void persistStatistics(boolean force) throws RemoteException, ACIDException {
+            MetadataTransactionContext ctx = null;
+            try {
+                long currentCount = adapter.getIngestedRecordsCount();
+                long diff = (currentCount - previousCount);
+                count.append(currentCount + ",");
+                rate.append(new Double(diff / period).intValue() + ",");
+                feedActivity.getFeedActivityDetails().put(FeedActivityDetails.INGESTION_RATE, "" + rate.toString());
+                feedActivity.getFeedActivityDetails().put(FeedActivityDetails.TOTAL_INGESTED, "" + count.toString());
+                numMeasurements++;
+                if (numMeasurements == batchPersistSize || force) {
+                    numMeasurements = 0;
+                    ctx = MetadataManager.INSTANCE.beginTransaction();
+                    MetadataManager.INSTANCE.registerFeedActivity(ctx, feedId, feedActivity);
+                    MetadataManager.INSTANCE.commitTransaction(ctx);
+                    count.delete(0, count.length() - 1);
+                    rate.delete(0, rate.length() - 1);
+                }
+                previousCount = currentCount;
+            } catch (Exception e) {
+                if (ctx != null) {
+                    MetadataManager.INSTANCE.abortTransaction(ctx);
+                }
+            }
+        }
+    }
+
+}
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/AdapterIdentifier.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/AdapterIdentifier.java
similarity index 96%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/AdapterIdentifier.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/AdapterIdentifier.java
index f6164ea..897faae 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/AdapterIdentifier.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/AdapterIdentifier.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.dataset.adapter;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.io.Serializable;
 
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/AlterFeedMessage.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/AlterFeedMessage.java
similarity index 83%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/AlterFeedMessage.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/AlterFeedMessage.java
index c36dc03..96f1dca 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/AlterFeedMessage.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/AlterFeedMessage.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.feed.lifecycle;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.util.Map;
 
@@ -25,9 +25,9 @@
 
     private static final long serialVersionUID = 1L;
 
-    private final Map<String, String> alteredConfParams;
+    private final Map<String, Object> alteredConfParams;
 
-    public AlterFeedMessage(Map<String, String> alteredConfParams) {
+    public AlterFeedMessage(Map<String, Object> alteredConfParams) {
         super(MessageType.ALTER);
         this.alteredConfParams = alteredConfParams;
     }
@@ -37,7 +37,7 @@
         return MessageType.ALTER;
     }
 
-    public Map<String, String> getAlteredConfParams() {
+    public Map<String, Object> getAlteredConfParams() {
         return alteredConfParams;
     }
 }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/BuiltinFeedPolicies.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/BuiltinFeedPolicies.java
new file mode 100644
index 0000000..9556ae0
--- /dev/null
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/BuiltinFeedPolicies.java
@@ -0,0 +1,83 @@
+package edu.uci.ics.asterix.metadata.feeds;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import edu.uci.ics.asterix.metadata.bootstrap.MetadataConstants;
+import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
+
+public class BuiltinFeedPolicies {
+
+    public static final FeedPolicy MISSTION_CRITICAL = initializeMissionCriticalFeedPolicy();
+
+    public static final FeedPolicy ADVANCED = initializeAdvancedFeedPolicy();
+
+    public static final FeedPolicy BASIC_MONITORED = initializeBasicMonitoredPolicy();
+
+    public static final FeedPolicy BASIC = initializeBasicPolicy();
+
+    public static final FeedPolicy[] policies = new FeedPolicy[] { MISSTION_CRITICAL, ADVANCED, BASIC_MONITORED, BASIC };
+
+    public static final FeedPolicy DEFAULT_POLICY = BASIC_MONITORED;
+
+    public static final String CONFIG_FEED_POLICY_KEY = "policy";
+
+    public static FeedPolicy getFeedPolicy(String policyName) {
+        for (FeedPolicy policy : policies) {
+            if (policy.getPolicyName().equalsIgnoreCase(policyName)) {
+                return policy;
+            }
+        }
+        return null;
+    }
+
+    private static FeedPolicy initializeMissionCriticalFeedPolicy() {
+        Map<String, String> policyParams = new HashMap<String, String>();
+        policyParams.put(FeedPolicyAccessor.APPLICATION_FAILURE_PERSIST_EXCEPTION, "true");
+        policyParams.put(FeedPolicyAccessor.APPLICATION_FAILURE_CONTINUE_ON_EXCEPTION, "true");
+        policyParams.put(FeedPolicyAccessor.HARDWARE_FAILURE_AUTO_RESTART, "true");
+        policyParams.put(FeedPolicyAccessor.STATISTICS_COLLECT, "true");
+        policyParams.put(FeedPolicyAccessor.STATISTICS_COLLECT_PERIOD, "60");
+        policyParams.put(FeedPolicyAccessor.STATISTICS_COLLECT_PERIOD_UNIT, FeedPolicyAccessor.TimeUnit.SEC.name());
+        policyParams.put(FeedPolicyAccessor.ELASTIC, "false");
+        String description = "MissionCritical";
+        return new FeedPolicy(MetadataConstants.METADATA_DATAVERSE_NAME, "MissionCritical", description, policyParams);
+    }
+
+    private static FeedPolicy initializeBasicPolicy() {
+        Map<String, String> policyParams = new HashMap<String, String>();
+        policyParams.put(FeedPolicyAccessor.APPLICATION_FAILURE_PERSIST_EXCEPTION, "false");
+        policyParams.put(FeedPolicyAccessor.APPLICATION_FAILURE_CONTINUE_ON_EXCEPTION, "false");
+        policyParams.put(FeedPolicyAccessor.HARDWARE_FAILURE_AUTO_RESTART, "false");
+        policyParams.put(FeedPolicyAccessor.STATISTICS_COLLECT, "false");
+        policyParams.put(FeedPolicyAccessor.ELASTIC, "false");
+        String description = "Basic";
+        return new FeedPolicy(MetadataConstants.METADATA_DATAVERSE_NAME, "Basic", description, policyParams);
+    }
+
+    private static FeedPolicy initializeBasicMonitoredPolicy() {
+        Map<String, String> policyParams = new HashMap<String, String>();
+        policyParams.put(FeedPolicyAccessor.APPLICATION_FAILURE_PERSIST_EXCEPTION, "true");
+        policyParams.put(FeedPolicyAccessor.APPLICATION_FAILURE_CONTINUE_ON_EXCEPTION, "false");
+        policyParams.put(FeedPolicyAccessor.HARDWARE_FAILURE_AUTO_RESTART, "false");
+        policyParams.put(FeedPolicyAccessor.STATISTICS_COLLECT, "true");
+        policyParams.put(FeedPolicyAccessor.STATISTICS_COLLECT_PERIOD, "5");
+        policyParams.put(FeedPolicyAccessor.STATISTICS_COLLECT_PERIOD_UNIT, FeedPolicyAccessor.TimeUnit.SEC.name());
+        policyParams.put(FeedPolicyAccessor.ELASTIC, "false");
+        String description = "BasicMonitored";
+        return new FeedPolicy(MetadataConstants.METADATA_DATAVERSE_NAME, "BasicMonitored", description, policyParams);
+    }
+
+    private static FeedPolicy initializeAdvancedFeedPolicy() {
+        Map<String, String> policyParams = new HashMap<String, String>();
+        policyParams.put(FeedPolicyAccessor.APPLICATION_FAILURE_PERSIST_EXCEPTION, "true");
+        policyParams.put(FeedPolicyAccessor.APPLICATION_FAILURE_CONTINUE_ON_EXCEPTION, "true");
+        policyParams.put(FeedPolicyAccessor.HARDWARE_FAILURE_AUTO_RESTART, "false");
+        policyParams.put(FeedPolicyAccessor.STATISTICS_COLLECT, "true");
+        policyParams.put(FeedPolicyAccessor.STATISTICS_COLLECT_PERIOD, "60");
+        policyParams.put(FeedPolicyAccessor.STATISTICS_COLLECT_PERIOD_UNIT, FeedPolicyAccessor.TimeUnit.SEC.name());
+        policyParams.put(FeedPolicyAccessor.ELASTIC, "false");
+        String description = "Advanced";
+        return new FeedPolicy(MetadataConstants.METADATA_DATAVERSE_NAME, "Advanced", description, policyParams);
+    }
+}
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/data/operator/ExternalDataScanOperatorDescriptor.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/ExternalDataScanOperatorDescriptor.java
similarity index 93%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/data/operator/ExternalDataScanOperatorDescriptor.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/ExternalDataScanOperatorDescriptor.java
index 4290132..07e1594 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/data/operator/ExternalDataScanOperatorDescriptor.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/ExternalDataScanOperatorDescriptor.java
@@ -12,12 +12,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.data.operator;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.util.Map;
 
-import edu.uci.ics.asterix.external.adapter.factory.IGenericDatasetAdapterFactory;
-import edu.uci.ics.asterix.external.dataset.adapter.IDatasourceAdapter;
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
 import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedActivityIdFactory.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedActivityIdFactory.java
new file mode 100644
index 0000000..db1a62a
--- /dev/null
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedActivityIdFactory.java
@@ -0,0 +1,25 @@
+package edu.uci.ics.asterix.metadata.feeds;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class FeedActivityIdFactory {
+	private static AtomicInteger id = new AtomicInteger();
+	private static boolean isInitialized = false;
+
+	public static boolean isInitialized() {
+		return isInitialized;
+	}
+
+	public static void initialize(int initialId) {
+		id.set(initialId);
+		isInitialized = true;
+	}
+
+	public static int generateFeedActivityId() {
+		return id.incrementAndGet();
+	}
+
+	public static int getMostRecentFeedActivityId() {
+		return id.get();
+	}
+}
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedFailureHandler.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedFailureHandler.java
new file mode 100644
index 0000000..77dbea0
--- /dev/null
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedFailureHandler.java
@@ -0,0 +1,81 @@
+package edu.uci.ics.asterix.metadata.feeds;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.event.schema.cluster.Node;
+import edu.uci.ics.asterix.metadata.cluster.ClusterManager;
+import edu.uci.ics.asterix.metadata.feeds.FeedLifecycleListener.FeedFailure;
+import edu.uci.ics.asterix.metadata.feeds.FeedLifecycleListener.FeedFailure.FailureType;
+import edu.uci.ics.asterix.metadata.feeds.FeedLifecycleListener.FeedFailureReport;
+import edu.uci.ics.asterix.metadata.feeds.FeedLifecycleListener.FeedInfo;
+import edu.uci.ics.asterix.om.util.AsterixClusterProperties;
+
+public class FeedFailureHandler implements Runnable {
+
+    private LinkedBlockingQueue<FeedFailureReport> inbox = null;
+
+    public FeedFailureHandler(LinkedBlockingQueue<FeedFailureReport> inbox) {
+        this.inbox = inbox;
+    }
+
+    @Override
+    public void run() {
+        while (true) {
+            try {
+                FeedFailureReport failureReport = inbox.take();
+                Map<String, Map<FeedInfo, List<FailureType>>> failureMap = new HashMap<String, Map<FeedInfo, List<FailureType>>>();
+                for (Map.Entry<FeedInfo, List<FeedFailure>> entry : failureReport.failures.entrySet()) {
+                    FeedInfo feedInfo = entry.getKey();
+                    List<FeedFailure> feedFailures = entry.getValue();
+                    for (FeedFailure feedFailure : feedFailures) {
+                        switch (feedFailure.failureType) {
+                            case COMPUTE_NODE:
+                            case INGESTION_NODE:
+                                Map<FeedInfo, List<FailureType>> failuresBecauseOfThisNode = failureMap
+                                        .get(feedFailure.nodeId);
+                                if (failuresBecauseOfThisNode == null) {
+                                    failuresBecauseOfThisNode = new HashMap<FeedInfo, List<FailureType>>();
+                                    failuresBecauseOfThisNode.put(feedInfo, new ArrayList<FailureType>());
+                                    failureMap.put(feedFailure.nodeId, failuresBecauseOfThisNode);
+                                }
+                                List<FailureType> feedF = failuresBecauseOfThisNode.get(feedInfo);
+                                if (feedF == null) {
+                                    feedF = new ArrayList<FailureType>();
+                                    failuresBecauseOfThisNode.put(feedInfo, feedF);
+                                }
+                                feedF.add(feedFailure.failureType);
+                                break;
+                            case STORAGE_NODE:
+                        }
+                    }
+                }
+
+                try {
+                    correctFailure(failureMap);
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }
+
+    }
+
+    private void correctFailure(Map<String, Map<FeedInfo, List<FailureType>>> failureMap) throws AsterixException {
+        for (String nodeId : failureMap.keySet()) {
+            Node node = AsterixClusterProperties.INSTANCE.getAvailableSubstitutionNode();
+            if (node != null) {
+                ClusterManager.INSTANCE.addNode(node);
+            }
+        }
+
+    }
+
+}
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/FeedId.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedId.java
similarity index 96%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/FeedId.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedId.java
index 94e679d..d56081d 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/FeedId.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedId.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.feed.lifecycle;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.io.Serializable;
 
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/data/operator/FeedIntakeOperatorDescriptor.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedIntakeOperatorDescriptor.java
similarity index 81%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/data/operator/FeedIntakeOperatorDescriptor.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedIntakeOperatorDescriptor.java
index a48cfb8..96be46a 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/data/operator/FeedIntakeOperatorDescriptor.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedIntakeOperatorDescriptor.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2009-2013 by The Regents of the University of California
+ * Copyright 2009-2012 by The Regents of the University of California
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * you may obtain a copy of the License from
@@ -12,15 +12,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.data.operator;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.util.Map;
 
-import edu.uci.ics.asterix.external.adapter.factory.IAdapterFactory;
-import edu.uci.ics.asterix.external.adapter.factory.IGenericDatasetAdapterFactory;
-import edu.uci.ics.asterix.external.adapter.factory.ITypedDatasetAdapterFactory;
-import edu.uci.ics.asterix.external.dataset.adapter.ITypedDatasourceAdapter;
-import edu.uci.ics.asterix.external.feed.lifecycle.FeedId;
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
@@ -43,18 +38,19 @@
     private final Map<String, Object> adapterConfiguration;
     private final IAType atype;
     private final FeedId feedId;
-    private final IAdapterFactory datasourceAdapterFactory;
+    private final Map<String, String> feedPolicy;
+
+    private transient IAdapterFactory datasourceAdapterFactory;
 
     public FeedIntakeOperatorDescriptor(JobSpecification spec, FeedId feedId, String adapter,
-            Map<String, Object> arguments, ARecordType atype, RecordDescriptor rDesc,
-            IAdapterFactory datasourceAdapterFactory) {
+            Map<String, Object> arguments, ARecordType atype, RecordDescriptor rDesc, Map<String, String> feedPolicy) {
         super(spec, 1, 1);
         recordDescriptors[0] = rDesc;
         this.adapterFactoryClassName = adapter;
         this.adapterConfiguration = arguments;
         this.atype = atype;
         this.feedId = feedId;
-        this.datasourceAdapterFactory = datasourceAdapterFactory;
+        this.feedPolicy = feedPolicy;
     }
 
     public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
@@ -62,6 +58,7 @@
             throws HyracksDataException {
         ITypedDatasourceAdapter adapter;
         try {
+            datasourceAdapterFactory = (IAdapterFactory) Class.forName(adapterFactoryClassName).newInstance();
             if (datasourceAdapterFactory instanceof IGenericDatasetAdapterFactory) {
                 adapter = (ITypedDatasourceAdapter) ((IGenericDatasetAdapterFactory) datasourceAdapterFactory)
                         .createAdapter(adapterConfiguration, atype);
@@ -75,6 +72,10 @@
         } catch (Exception e) {
             throw new HyracksDataException("initialization of adapter failed", e);
         }
-        return new FeedIntakeOperatorNodePushable(feedId, adapter, partition);
+        return new FeedIntakeOperatorNodePushable(feedId, adapter, feedPolicy, partition);
+    }
+
+    public FeedId getFeedId() {
+        return feedId;
     }
 }
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/data/operator/FeedIntakeOperatorNodePushable.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedIntakeOperatorNodePushable.java
similarity index 81%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/data/operator/FeedIntakeOperatorNodePushable.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedIntakeOperatorNodePushable.java
index 31470f3..f6bde26 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/data/operator/FeedIntakeOperatorNodePushable.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedIntakeOperatorNodePushable.java
@@ -12,19 +12,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.data.operator;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.nio.ByteBuffer;
+import java.util.Map;
 import java.util.concurrent.LinkedBlockingQueue;
 
 import edu.uci.ics.asterix.common.api.AsterixThreadExecutor;
-import edu.uci.ics.asterix.external.dataset.adapter.IDatasourceAdapter;
-import edu.uci.ics.asterix.external.feed.lifecycle.AlterFeedMessage;
-import edu.uci.ics.asterix.external.feed.lifecycle.FeedId;
-import edu.uci.ics.asterix.external.feed.lifecycle.FeedManager;
-import edu.uci.ics.asterix.external.feed.lifecycle.IFeedManager;
-import edu.uci.ics.asterix.external.feed.lifecycle.IFeedMessage;
-import edu.uci.ics.asterix.feed.managed.adapter.IManagedFeedAdapter;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
 import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
 
@@ -39,13 +33,19 @@
     private final FeedId feedId;
     private final LinkedBlockingQueue<IFeedMessage> inbox;
     private FeedInboxMonitor feedInboxMonitor;
+    private final Map<String, String> feedPolicy;
+    private final FeedPolicyEnforcer policyEnforcer;
 
-    public FeedIntakeOperatorNodePushable(FeedId feedId, IDatasourceAdapter adapter, int partition) {
+    public FeedIntakeOperatorNodePushable(FeedId feedId, IDatasourceAdapter adapter, Map<String, String> feedPolicy,
+            int partition) {
         this.adapter = adapter;
         this.partition = partition;
         this.feedManager = (IFeedManager) FeedManager.INSTANCE;
         this.feedId = feedId;
         inbox = new LinkedBlockingQueue<IFeedMessage>();
+        this.feedPolicy = feedPolicy;
+        policyEnforcer = new FeedPolicyEnforcer(feedId, feedPolicy);
+
     }
 
     @Override
@@ -57,19 +57,19 @@
         }
         writer.open();
         try {
+            ((AbstractFeedDatasourceAdapter) adapter).setFeedPolicyEnforcer(policyEnforcer);
             adapter.start(partition, writer);
         } catch (Exception e) {
             e.printStackTrace();
             throw new HyracksDataException(e);
-            /* 
-             we do not throw an exception, but allow the operator to close
-             gracefully throwing an exception here would result in a job abort and a
-             transaction roll back that undoes all the work done so far.
-             */
-
         } finally {
             writer.close();
             if (adapter instanceof IManagedFeedAdapter) {
+                try {
+                    ((IManagedFeedAdapter) adapter).stop();
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
                 feedManager.unregisterFeedMsgQueue(feedId, inbox);
             }
         }
@@ -82,13 +82,17 @@
 
     @Override
     public void close() throws HyracksDataException {
-        
+
     }
 
     @Override
     public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
         // do nothing
     }
+
+    public Map<String, String> getFeedPolicy() {
+        return feedPolicy;
+    }
 }
 
 class FeedInboxMonitor extends Thread {
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedJobLifecycleListener.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedJobLifecycleListener.java
new file mode 100644
index 0000000..3a981b2
--- /dev/null
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedJobLifecycleListener.java
@@ -0,0 +1,289 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.metadata.feeds;
+
+import java.io.Serializable;
+import java.rmi.RemoteException;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import edu.uci.ics.asterix.common.exceptions.ACIDException;
+import edu.uci.ics.asterix.metadata.MetadataException;
+import edu.uci.ics.asterix.metadata.MetadataManager;
+import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity.FeedActivityType;
+import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.AssignRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.EmptyTupleSourceRuntimeFactory;
+import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.job.IActivityClusterGraphGenerator;
+import edu.uci.ics.hyracks.api.job.IActivityClusterGraphGeneratorFactory;
+import edu.uci.ics.hyracks.api.job.IJobLifecycleListener;
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobInfo;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+
+//import edu.uci.ics.hyracks.api.job.JobInfo;
+
+public class FeedJobLifecycleListener implements IJobLifecycleListener, Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    public static FeedJobLifecycleListener INSTANCE = new FeedJobLifecycleListener();
+
+    private LinkedBlockingQueue<Message> inbox;
+
+    private FeedJobLifecycleListener() {
+        inbox = new LinkedBlockingQueue<Message>();
+        feedJobNotificationHandler = new FeedJobNotificationHandler(inbox);
+        new Thread(feedJobNotificationHandler).start();
+    }
+
+    private final FeedJobNotificationHandler feedJobNotificationHandler;
+
+    @Override
+    public void notifyJobStart(JobId jobId) throws HyracksException {
+        if (feedJobNotificationHandler.isRegisteredFeed(jobId)) {
+            inbox.add(new Message(jobId, Message.MessageKind.JOB_START));
+        }
+    }
+
+    @Override
+    public void notifyJobFinish(JobId jobId) throws HyracksException {
+        if (feedJobNotificationHandler.isRegisteredFeed(jobId)) {
+            inbox.add(new Message(jobId, Message.MessageKind.JOB_FINISH));
+        }
+    }
+
+    @Override
+    public void notifyJobCreation(JobId jobId, IActivityClusterGraphGeneratorFactory acggf) throws HyracksException {
+
+        IActivityClusterGraphGenerator acgg = acggf.createActivityClusterGraphGenerator(jobId, AsterixAppContextInfo
+                .getInstance().getCCApplicationContext(), EnumSet.noneOf(JobFlag.class));
+        JobSpecification spec = acggf.getJobSpecification();
+        boolean feedIngestionJob = false;
+        FeedId feedId = null;
+        for (IOperatorDescriptor opDesc : spec.getOperatorMap().values()) {
+            if (!(opDesc instanceof FeedIntakeOperatorDescriptor)) {
+                continue;
+            }
+            feedId = ((FeedIntakeOperatorDescriptor) opDesc).getFeedId();
+            feedIngestionJob = true;
+            break;
+        }
+        if (feedIngestionJob) {
+            feedJobNotificationHandler.registerFeed(feedId, jobId, spec);
+        }
+
+    }
+
+    private static class Message {
+        public JobId jobId;
+
+        public enum MessageKind {
+            JOB_START,
+            JOB_FINISH
+        }
+
+        public MessageKind messageKind;
+
+        public Message(JobId jobId, MessageKind msgKind) {
+            this.jobId = jobId;
+            this.messageKind = msgKind;
+        }
+    }
+
+    private static class FeedJobNotificationHandler implements Runnable, Serializable {
+
+        private static final long serialVersionUID = 1L;
+        private LinkedBlockingQueue<Message> inbox;
+        private Map<JobId, FeedInfo> registeredFeeds = new HashMap<JobId, FeedInfo>();
+
+        public FeedJobNotificationHandler(LinkedBlockingQueue<Message> inbox) {
+            this.inbox = inbox;
+        }
+
+        public boolean isRegisteredFeed(JobId jobId) {
+            return registeredFeeds.containsKey(jobId);
+        }
+
+        public void registerFeed(FeedId feedId, JobId jobId, JobSpecification jobSpec) {
+            if (registeredFeeds.containsKey(jobId)) {
+                throw new IllegalStateException(" Feed already registered ");
+            }
+            registeredFeeds.put(jobId, new FeedInfo(feedId, jobSpec));
+        }
+
+        @Override
+        public void run() {
+            Message mesg;
+            while (true) {
+                try {
+                    mesg = inbox.take();
+                    FeedInfo feedInfo = registeredFeeds.get(mesg.jobId);
+                    switch (mesg.messageKind) {
+                        case JOB_START:
+                            handleJobStartMessage(feedInfo, mesg);
+                            break;
+                        case JOB_FINISH:
+                            handleJobFinishMessage(feedInfo, mesg);
+                            break;
+                    }
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+
+            }
+        }
+
+        private void handleJobStartMessage(FeedInfo feedInfo, Message message) {
+
+            JobSpecification jobSpec = feedInfo.jobSpec;
+
+            List<OperatorDescriptorId> ingestOperatorIds = new ArrayList<OperatorDescriptorId>();
+            List<OperatorDescriptorId> computeOperatorIds = new ArrayList<OperatorDescriptorId>();
+
+            Map<OperatorDescriptorId, IOperatorDescriptor> operators = jobSpec.getOperatorMap();
+            for (Entry<OperatorDescriptorId, IOperatorDescriptor> entry : operators.entrySet()) {
+                if (entry.getValue() instanceof AlgebricksMetaOperatorDescriptor) {
+                    AlgebricksMetaOperatorDescriptor op = ((AlgebricksMetaOperatorDescriptor) entry.getValue());
+                    IPushRuntimeFactory[] runtimeFactories = op.getPipeline().getRuntimeFactories();
+                    for (IPushRuntimeFactory rf : runtimeFactories) {
+                        if (rf instanceof EmptyTupleSourceRuntimeFactory) {
+                            ingestOperatorIds.add(entry.getKey());
+                        } else if (rf instanceof AssignRuntimeFactory) {
+                            computeOperatorIds.add(entry.getKey());
+                        }
+                    }
+                }
+            }
+
+            try {
+                IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc();
+                JobInfo info = hcc.getJobInfo(message.jobId);
+
+                Map<String, String> feedActivityDetails = new HashMap<String, String>();
+                StringBuilder ingestLocs = new StringBuilder();
+                for (OperatorDescriptorId ingestOpId : ingestOperatorIds) {
+                    feedInfo.ingestLocations.addAll(info.getOperatorLocations().get(ingestOpId));
+                }
+                StringBuilder computeLocs = new StringBuilder();
+                for (OperatorDescriptorId computeOpId : computeOperatorIds) {
+                    List<String> locations = info.getOperatorLocations().get(computeOpId);
+                    if (locations != null) {
+                        feedInfo.computeLocations.addAll(locations);
+                    } else {
+                        feedInfo.computeLocations.addAll(feedInfo.ingestLocations);
+                    }
+                }
+
+                for (String ingestLoc : feedInfo.ingestLocations) {
+                    ingestLocs.append(ingestLoc);
+                    ingestLocs.append(",");
+                }
+                for (String computeLoc : feedInfo.computeLocations) {
+                    computeLocs.append(computeLoc);
+                    computeLocs.append(",");
+                }
+
+                feedActivityDetails.put(FeedActivity.FeedActivityDetails.INGEST_LOCATIONS, ingestLocs.toString());
+                feedActivityDetails.put(FeedActivity.FeedActivityDetails.COMPUTE_LOCATIONS, computeLocs.toString());
+
+                FeedActivity feedActivity = new FeedActivity(feedInfo.feedId.getDataverse(),
+                        feedInfo.feedId.getDataset(), FeedActivityType.FEED_BEGIN, feedActivityDetails);
+
+                MetadataManager.INSTANCE.acquireWriteLatch();
+                MetadataTransactionContext mdTxnCtx = null;
+                try {
+                    mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                    MetadataManager.INSTANCE.registerFeedActivity(mdTxnCtx, new FeedId(feedInfo.feedId.getDataverse(),
+                            feedInfo.feedId.getDataset()), feedActivity);
+                    MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                } catch (Exception e) {
+                    MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
+                } finally {
+                    MetadataManager.INSTANCE.releaseWriteLatch();
+                }
+            } catch (Exception e) {
+                // TODO Add Exception handling here
+            }
+
+        }
+
+        private void handleJobFinishMessage(FeedInfo feedInfo, Message message) {
+
+            MetadataManager.INSTANCE.acquireWriteLatch();
+            MetadataTransactionContext mdTxnCtx = null;
+
+            try {
+                IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc();
+                JobInfo info = hcc.getJobInfo(message.jobId);
+                JobStatus status = info.getPendingStatus();
+                List<Exception> exceptions;
+                boolean failure = status != null && status.equals(JobStatus.FAILURE);
+                FeedActivityType activityType = FeedActivityType.FEED_END;
+                Map<String, String> details = new HashMap<String, String>();
+                if (failure) {
+                    exceptions = info.getPendingExceptions();
+                    activityType = FeedActivityType.FEED_FAILURE;
+                    details.put(FeedActivity.FeedActivityDetails.EXCEPTION_MESSAGE, exceptions.get(0).getMessage());
+                }
+                mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                FeedActivity feedActivity = new FeedActivity(feedInfo.feedId.getDataverse(),
+                        feedInfo.feedId.getDataset(), activityType, details);
+                MetadataManager.INSTANCE.registerFeedActivity(mdTxnCtx, new FeedId(feedInfo.feedId.getDataverse(),
+                        feedInfo.feedId.getDataset()), feedActivity);
+                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+            } catch (RemoteException | ACIDException | MetadataException e) {
+                try {
+                    MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
+                } catch (RemoteException | ACIDException ae) {
+                    throw new IllegalStateException(" Unable to abort ");
+                }
+            } catch (Exception e) {
+                // add exception handling here
+            } finally {
+                MetadataManager.INSTANCE.releaseWriteLatch();
+            }
+
+        }
+    }
+
+    private static class FeedInfo {
+        public FeedId feedId;
+        public JobSpecification jobSpec;
+        public List<String> ingestLocations = new ArrayList<String>();
+        public List<String> computeLocations = new ArrayList<String>();
+
+        public FeedInfo(FeedId feedId, JobSpecification jobSpec) {
+            this.feedId = feedId;
+            this.jobSpec = jobSpec;
+        }
+
+    }
+}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedLifecycleListener.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedLifecycleListener.java
new file mode 100644
index 0000000..ffa6c7e
--- /dev/null
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedLifecycleListener.java
@@ -0,0 +1,364 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.metadata.feeds;
+
+import java.io.Serializable;
+import java.rmi.RemoteException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import edu.uci.ics.asterix.common.exceptions.ACIDException;
+import edu.uci.ics.asterix.metadata.MetadataException;
+import edu.uci.ics.asterix.metadata.MetadataManager;
+import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity.FeedActivityType;
+import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.AssignRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.EmptyTupleSourceRuntimeFactory;
+import edu.uci.ics.hyracks.api.application.IClusterLifecycleListener;
+import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.job.IActivityClusterGraphGenerator;
+import edu.uci.ics.hyracks.api.job.IActivityClusterGraphGeneratorFactory;
+import edu.uci.ics.hyracks.api.job.IJobLifecycleListener;
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobInfo;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+
+//import edu.uci.ics.hyracks.api.job.JobInfo;
+
+public class FeedLifecycleListener implements IJobLifecycleListener, IClusterLifecycleListener, Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    public static FeedLifecycleListener INSTANCE = new FeedLifecycleListener();
+
+    private LinkedBlockingQueue<Message> jobEventInbox;
+    private LinkedBlockingQueue<FeedFailureReport> failureEventInbox;
+
+    private FeedLifecycleListener() {
+        jobEventInbox = new LinkedBlockingQueue<Message>();
+        feedJobNotificationHandler = new FeedJobNotificationHandler(jobEventInbox);
+        failureEventInbox = new LinkedBlockingQueue<FeedFailureReport>();
+        feedFailureNotificationHandler = new FeedFailureHandler(failureEventInbox);
+        new Thread(feedJobNotificationHandler).start();
+        new Thread(feedFailureNotificationHandler).start();
+    }
+
+    private final FeedJobNotificationHandler feedJobNotificationHandler;
+    private final FeedFailureHandler feedFailureNotificationHandler;
+
+    @Override
+    public void notifyJobStart(JobId jobId) throws HyracksException {
+        if (feedJobNotificationHandler.isRegisteredFeed(jobId)) {
+            jobEventInbox.add(new Message(jobId, Message.MessageKind.JOB_START));
+        }
+    }
+
+    @Override
+    public void notifyJobFinish(JobId jobId) throws HyracksException {
+        if (feedJobNotificationHandler.isRegisteredFeed(jobId)) {
+            jobEventInbox.add(new Message(jobId, Message.MessageKind.JOB_FINISH));
+        }
+    }
+
+    @Override
+    public void notifyJobCreation(JobId jobId, IActivityClusterGraphGeneratorFactory acggf) throws HyracksException {
+
+        IActivityClusterGraphGenerator acgg = acggf.createActivityClusterGraphGenerator(jobId, AsterixAppContextInfo
+                .getInstance().getCCApplicationContext(), EnumSet.noneOf(JobFlag.class));
+        JobSpecification spec = acggf.getJobSpecification();
+        boolean feedIngestionJob = false;
+        FeedId feedId = null;
+        for (IOperatorDescriptor opDesc : spec.getOperatorMap().values()) {
+            if (!(opDesc instanceof FeedIntakeOperatorDescriptor)) {
+                continue;
+            }
+            feedId = ((FeedIntakeOperatorDescriptor) opDesc).getFeedId();
+            feedIngestionJob = true;
+            break;
+        }
+        if (feedIngestionJob) {
+            feedJobNotificationHandler.registerFeed(feedId, jobId, spec);
+        }
+
+    }
+
+    private static class Message {
+        public JobId jobId;
+
+        public enum MessageKind {
+            JOB_START,
+            JOB_FINISH
+        }
+
+        public MessageKind messageKind;
+
+        public Message(JobId jobId, MessageKind msgKind) {
+            this.jobId = jobId;
+            this.messageKind = msgKind;
+        }
+    }
+
+    public static class FeedFailureReport {
+        public Map<FeedInfo, List<FeedFailure>> failures = new HashMap<FeedInfo, List<FeedFailure>>();
+
+        @Override
+        public String toString() {
+            StringBuilder builder = new StringBuilder();
+            for (Map.Entry<FeedLifecycleListener.FeedInfo, List<FeedLifecycleListener.FeedFailure>> entry : failures
+                    .entrySet()) {
+                builder.append(entry.getKey() + " -> failures");
+                for (FeedFailure failure : entry.getValue()) {
+                    builder.append("failure -> " + failure);
+                }
+            }
+            return builder.toString();
+        }
+    }
+
+    private static class FeedJobNotificationHandler implements Runnable, Serializable {
+
+        private static final long serialVersionUID = 1L;
+        private LinkedBlockingQueue<Message> inbox;
+        private Map<JobId, FeedInfo> registeredFeeds = new HashMap<JobId, FeedInfo>();
+
+        public FeedJobNotificationHandler(LinkedBlockingQueue<Message> inbox) {
+            this.inbox = inbox;
+        }
+
+        public boolean isRegisteredFeed(JobId jobId) {
+            return registeredFeeds.containsKey(jobId);
+        }
+
+        public void registerFeed(FeedId feedId, JobId jobId, JobSpecification jobSpec) {
+            if (registeredFeeds.containsKey(jobId)) {
+                throw new IllegalStateException(" Feed already registered ");
+            }
+            registeredFeeds.put(jobId, new FeedInfo(feedId, jobSpec));
+        }
+
+        @Override
+        public void run() {
+            Message mesg;
+            while (true) {
+                try {
+                    mesg = inbox.take();
+                    FeedInfo feedInfo = registeredFeeds.get(mesg.jobId);
+                    switch (mesg.messageKind) {
+                        case JOB_START:
+                            handleJobStartMessage(feedInfo, mesg);
+                            break;
+                        case JOB_FINISH:
+                            handleJobFinishMessage(feedInfo, mesg);
+                            break;
+                    }
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+
+            }
+        }
+
+        private void handleJobStartMessage(FeedInfo feedInfo, Message message) {
+
+            JobSpecification jobSpec = feedInfo.jobSpec;
+
+            List<OperatorDescriptorId> ingestOperatorIds = new ArrayList<OperatorDescriptorId>();
+            List<OperatorDescriptorId> computeOperatorIds = new ArrayList<OperatorDescriptorId>();
+
+            Map<OperatorDescriptorId, IOperatorDescriptor> operators = jobSpec.getOperatorMap();
+            for (Entry<OperatorDescriptorId, IOperatorDescriptor> entry : operators.entrySet()) {
+                if (entry.getValue() instanceof AlgebricksMetaOperatorDescriptor) {
+                    AlgebricksMetaOperatorDescriptor op = ((AlgebricksMetaOperatorDescriptor) entry.getValue());
+                    IPushRuntimeFactory[] runtimeFactories = op.getPipeline().getRuntimeFactories();
+                    for (IPushRuntimeFactory rf : runtimeFactories) {
+                        if (rf instanceof EmptyTupleSourceRuntimeFactory) {
+                            ingestOperatorIds.add(entry.getKey());
+                        } else if (rf instanceof AssignRuntimeFactory) {
+                            computeOperatorIds.add(entry.getKey());
+                        }
+                    }
+                }
+            }
+
+            try {
+                IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc();
+                JobInfo info = hcc.getJobInfo(message.jobId);
+
+                Map<String, String> feedActivityDetails = new HashMap<String, String>();
+                StringBuilder ingestLocs = new StringBuilder();
+                for (OperatorDescriptorId ingestOpId : ingestOperatorIds) {
+                    feedInfo.ingestLocations.addAll(info.getOperatorLocations().get(ingestOpId));
+                }
+                StringBuilder computeLocs = new StringBuilder();
+                for (OperatorDescriptorId computeOpId : computeOperatorIds) {
+                    List<String> locations = info.getOperatorLocations().get(computeOpId);
+                    if (locations != null) {
+                        feedInfo.computeLocations.addAll(locations);
+                    } else {
+                        feedInfo.computeLocations.addAll(feedInfo.ingestLocations);
+                    }
+                }
+
+                for (String ingestLoc : feedInfo.ingestLocations) {
+                    ingestLocs.append(ingestLoc);
+                    ingestLocs.append(",");
+                }
+                for (String computeLoc : feedInfo.computeLocations) {
+                    computeLocs.append(computeLoc);
+                    computeLocs.append(",");
+                }
+
+                feedActivityDetails.put(FeedActivity.FeedActivityDetails.INGEST_LOCATIONS, ingestLocs.toString());
+                feedActivityDetails.put(FeedActivity.FeedActivityDetails.COMPUTE_LOCATIONS, computeLocs.toString());
+
+                FeedActivity feedActivity = new FeedActivity(feedInfo.feedId.getDataverse(),
+                        feedInfo.feedId.getDataset(), FeedActivityType.FEED_BEGIN, feedActivityDetails);
+
+                MetadataManager.INSTANCE.acquireWriteLatch();
+                MetadataTransactionContext mdTxnCtx = null;
+                try {
+                    mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                    MetadataManager.INSTANCE.registerFeedActivity(mdTxnCtx, new FeedId(feedInfo.feedId.getDataverse(),
+                            feedInfo.feedId.getDataset()), feedActivity);
+                    MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                } catch (Exception e) {
+                    MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
+                } finally {
+                    MetadataManager.INSTANCE.releaseWriteLatch();
+                }
+            } catch (Exception e) {
+                // TODO Add Exception handling here
+            }
+
+        }
+
+        private void handleJobFinishMessage(FeedInfo feedInfo, Message message) {
+
+            MetadataManager.INSTANCE.acquireWriteLatch();
+            MetadataTransactionContext mdTxnCtx = null;
+
+            try {
+                IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc();
+                JobInfo info = hcc.getJobInfo(message.jobId);
+                JobStatus status = info.getPendingStatus();
+                List<Exception> exceptions;
+                boolean failure = status != null && status.equals(JobStatus.FAILURE);
+                FeedActivityType activityType = FeedActivityType.FEED_END;
+                Map<String, String> details = new HashMap<String, String>();
+                if (failure) {
+                    exceptions = info.getPendingExceptions();
+                    activityType = FeedActivityType.FEED_FAILURE;
+                    details.put(FeedActivity.FeedActivityDetails.EXCEPTION_MESSAGE, exceptions.get(0).getMessage());
+                }
+                mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                FeedActivity feedActivity = new FeedActivity(feedInfo.feedId.getDataverse(),
+                        feedInfo.feedId.getDataset(), activityType, details);
+                MetadataManager.INSTANCE.registerFeedActivity(mdTxnCtx, new FeedId(feedInfo.feedId.getDataverse(),
+                        feedInfo.feedId.getDataset()), feedActivity);
+                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+            } catch (RemoteException | ACIDException | MetadataException e) {
+                try {
+                    MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
+                } catch (RemoteException | ACIDException ae) {
+                    throw new IllegalStateException(" Unable to abort ");
+                }
+            } catch (Exception e) {
+                // add exception handling here
+            } finally {
+                MetadataManager.INSTANCE.releaseWriteLatch();
+            }
+
+        }
+    }
+
+    public static class FeedInfo {
+        public FeedId feedId;
+        public JobSpecification jobSpec;
+        public List<String> ingestLocations = new ArrayList<String>();
+        public List<String> computeLocations = new ArrayList<String>();
+
+        public FeedInfo(FeedId feedId, JobSpecification jobSpec) {
+            this.feedId = feedId;
+            this.jobSpec = jobSpec;
+        }
+
+    }
+
+    @Override
+    public void notifyNodeJoin(String nodeId, Map<String, String> ncConfiguration) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public void notifyNodeFailure(Set<String> deadNodeIds) {
+        Collection<FeedInfo> feedInfos = feedJobNotificationHandler.registeredFeeds.values();
+        FeedFailureReport failureReport = new FeedFailureReport();
+        for (FeedInfo feedInfo : feedInfos) {
+            for (String deadNodeId : deadNodeIds) {
+                if (feedInfo.ingestLocations.contains(deadNodeId)) {
+                    List<FeedFailure> failures = failureReport.failures.get(feedInfo);
+                    if (failures == null) {
+                        failures = new ArrayList<FeedFailure>();
+                        failureReport.failures.put(feedInfo, failures);
+                    }
+                    failures.add(new FeedFailure(FeedFailure.FailureType.INGESTION_NODE, deadNodeId));
+                }
+                if (feedInfo.computeLocations.contains(deadNodeId)) {
+                    List<FeedFailure> failures = failureReport.failures.get(feedInfo);
+                    if (failures == null) {
+                        failures = new ArrayList<FeedFailure>();
+                        failureReport.failures.put(feedInfo, failures);
+                    }
+                    failures.add(new FeedFailure(FeedFailure.FailureType.COMPUTE_NODE, deadNodeId));
+                }
+            }
+        }
+        failureEventInbox.add(failureReport);
+    }
+
+    public static class FeedFailure {
+
+        public enum FailureType {
+            INGESTION_NODE,
+            COMPUTE_NODE,
+            STORAGE_NODE
+        }
+
+        public FailureType failureType;
+        public String nodeId;
+
+        public FeedFailure(FailureType failureType, String nodeId) {
+            this.failureType = failureType;
+            this.nodeId = nodeId;
+        }
+    }
+}
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/FeedManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedManager.java
similarity index 91%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/FeedManager.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedManager.java
index 5b3ed35..f9892f2 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/FeedManager.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedManager.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.feed.lifecycle;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.util.HashMap;
 import java.util.HashSet;
@@ -21,6 +21,9 @@
 import java.util.concurrent.LinkedBlockingQueue;
 
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.metadata.feeds.FeedId;
+import edu.uci.ics.asterix.metadata.feeds.IFeedManager;
+import edu.uci.ics.asterix.metadata.feeds.IFeedMessage;
 
 /**
  * Handle (de-)registration of feeds for delivery of control messages.
@@ -43,7 +46,7 @@
                 for (LinkedBlockingQueue<IFeedMessage> queue : operatorQueues) {
                     queue.put(feedMessage);
                 }
-            } 
+            }
         } catch (Exception e) {
             throw new AsterixException(e);
         }
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/FeedMessage.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessage.java
similarity index 91%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/FeedMessage.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessage.java
index 96262fe..ae630ac 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/FeedMessage.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessage.java
@@ -12,7 +12,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.feed.lifecycle;
+package edu.uci.ics.asterix.metadata.feeds;
+
+import edu.uci.ics.asterix.metadata.feeds.IFeedMessage;
 
 /**
  * A control message that can be sent to the runtime instance of a 
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/data/operator/FeedMessageOperatorDescriptor.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageOperatorDescriptor.java
similarity index 91%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/data/operator/FeedMessageOperatorDescriptor.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageOperatorDescriptor.java
index 300e3af..677fc40 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/data/operator/FeedMessageOperatorDescriptor.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageOperatorDescriptor.java
@@ -12,12 +12,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.data.operator;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.util.List;
 
-import edu.uci.ics.asterix.external.feed.lifecycle.FeedId;
-import edu.uci.ics.asterix.external.feed.lifecycle.IFeedMessage;
 import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
 import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
 import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/data/operator/FeedMessageOperatorNodePushable.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageOperatorNodePushable.java
similarity index 86%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/data/operator/FeedMessageOperatorNodePushable.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageOperatorNodePushable.java
index b6bacef..be5c8f9 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/data/operator/FeedMessageOperatorNodePushable.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageOperatorNodePushable.java
@@ -12,15 +12,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.data.operator;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.util.ArrayList;
 import java.util.List;
 
-import edu.uci.ics.asterix.external.feed.lifecycle.FeedId;
-import edu.uci.ics.asterix.external.feed.lifecycle.FeedManager;
-import edu.uci.ics.asterix.external.feed.lifecycle.IFeedManager;
-import edu.uci.ics.asterix.external.feed.lifecycle.IFeedMessage;
 import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
 import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedPolicyAccessor.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedPolicyAccessor.java
new file mode 100644
index 0000000..3e16877
--- /dev/null
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedPolicyAccessor.java
@@ -0,0 +1,81 @@
+package edu.uci.ics.asterix.metadata.feeds;
+
+import java.util.Map;
+
+public class FeedPolicyAccessor {
+    public static final String APPLICATION_FAILURE_PERSIST_EXCEPTION = "software.failure.persist.exception";
+    public static final String APPLICATION_FAILURE_CONTINUE_ON_EXCEPTION = "software.failure.continue.on.exception";
+    public static final String HARDWARE_FAILURE_AUTO_RESTART = "hardware.failure.auto.restart";
+    public static final String STATISTICS_COLLECT = "statistics.collect";
+    public static final String STATISTICS_COLLECT_PERIOD = "statistics.collect.period";
+    public static final String STATISTICS_COLLECT_PERIOD_UNIT = "statistics.collect.period.unit";
+    public static final String ELASTIC = "elastic";
+
+    public enum TimeUnit {
+        SEC,
+        MIN,
+        HRS,
+        DAYS
+    }
+
+    private Map<String, String> feedPolicy;
+
+    public FeedPolicyAccessor(Map<String, String> feedPolicy) {
+        this.feedPolicy = feedPolicy;
+    }
+
+    public boolean persistExceptionDetailsOnApplicationFailure() {
+        return getBooleanPropertyValue(APPLICATION_FAILURE_PERSIST_EXCEPTION);
+    }
+
+    public boolean continueOnApplicationFailure() {
+        return getBooleanPropertyValue(APPLICATION_FAILURE_CONTINUE_ON_EXCEPTION);
+    }
+
+    public boolean autoRestartOnHardwareFailure() {
+        return getBooleanPropertyValue(HARDWARE_FAILURE_AUTO_RESTART);
+    }
+
+    public boolean collectStatistics() {
+        return getBooleanPropertyValue(STATISTICS_COLLECT);
+    }
+
+    public long getStatisicsCollectionPeriodInSecs() {
+        return getIntegerPropertyValue(STATISTICS_COLLECT_PERIOD) * getTimeUnitFactor();
+    }
+
+    public boolean isElastic() {
+        return getBooleanPropertyValue(ELASTIC);
+    }
+
+    private int getTimeUnitFactor() {
+        String v = feedPolicy.get(STATISTICS_COLLECT_PERIOD_UNIT);
+        int factor = 1;
+        switch (TimeUnit.valueOf(v)) {
+            case SEC:
+                factor = 1;
+                break;
+            case MIN:
+                factor = 60;
+                break;
+            case HRS:
+                factor = 3600;
+                break;
+            case DAYS:
+                factor = 216000;
+                break;
+
+        }
+        return factor;
+    }
+
+    private boolean getBooleanPropertyValue(String key) {
+        String v = feedPolicy.get(key);
+        return v == null ? false : Boolean.valueOf(v);
+    }
+
+    private int getIntegerPropertyValue(String key) {
+        String v = feedPolicy.get(key);
+        return Integer.parseInt(v);
+    }
+}
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedPolicyEnforcer.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedPolicyEnforcer.java
new file mode 100644
index 0000000..f708c91
--- /dev/null
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedPolicyEnforcer.java
@@ -0,0 +1,64 @@
+package edu.uci.ics.asterix.metadata.feeds;
+
+import java.rmi.RemoteException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.asterix.common.exceptions.ACIDException;
+import edu.uci.ics.asterix.metadata.MetadataManager;
+import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity;
+import edu.uci.ics.asterix.metadata.entities.FeedActivity.FeedActivityType;
+
+public class FeedPolicyEnforcer {
+
+    private final FeedId feedId;
+    private final FeedPolicyAccessor feedPolicyAccessor;
+    private final FeedActivity feedActivity;
+
+    public FeedPolicyEnforcer(FeedId feedId, Map<String, String> feedPolicy) {
+        this.feedId = feedId;
+        this.feedPolicyAccessor = new FeedPolicyAccessor(feedPolicy);
+        this.feedActivity = new FeedActivity(feedId.getDataverse(), feedId.getDataset(), null,
+                new HashMap<String, String>());
+    }
+
+    public boolean handleSoftwareFailure(Exception e) throws RemoteException, ACIDException {
+        boolean continueIngestion = feedPolicyAccessor.continueOnApplicationFailure();
+        if (feedPolicyAccessor.persistExceptionDetailsOnApplicationFailure()) {
+            persistExceptionDetails(e);
+        }
+        return continueIngestion;
+    }
+
+    private synchronized void persistExceptionDetails(Exception e) throws RemoteException, ACIDException {
+        MetadataManager.INSTANCE.acquireWriteLatch();
+        MetadataTransactionContext ctx = null;
+        try {
+            ctx = MetadataManager.INSTANCE.beginTransaction();
+            feedActivity.setActivityType(FeedActivityType.FEED_FAILURE);
+            feedActivity.getFeedActivityDetails().put(FeedActivity.FeedActivityDetails.EXCEPTION_MESSAGE,
+                    e.getMessage());
+            MetadataManager.INSTANCE.registerFeedActivity(ctx, feedId, feedActivity);
+            MetadataManager.INSTANCE.commitTransaction(ctx);
+        } catch (Exception e2) {
+            MetadataManager.INSTANCE.abortTransaction(ctx);
+        } finally {
+            MetadataManager.INSTANCE.releaseWriteLatch();
+        }
+    }
+
+    public void handleHardwareFailure(List<String> nodeId) {
+
+    }
+
+    public FeedPolicyAccessor getFeedPolicyAccessor() {
+        return feedPolicyAccessor;
+    }
+
+    public FeedId getFeedId() {
+        return feedId;
+    }
+
+}
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/IAdapterFactory.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IAdapterFactory.java
similarity index 94%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/IAdapterFactory.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IAdapterFactory.java
index 5cb6777..85e09d4 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/IAdapterFactory.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IAdapterFactory.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.adapter.factory;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.io.Serializable;
 
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/IDatasourceAdapter.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IDatasourceAdapter.java
similarity index 98%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/IDatasourceAdapter.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IDatasourceAdapter.java
index 9403bfe..a117f8a 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/IDatasourceAdapter.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IDatasourceAdapter.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.dataset.adapter;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.io.Serializable;
 import java.util.Map;
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/IFeedManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IFeedManager.java
similarity index 97%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/IFeedManager.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IFeedManager.java
index 2febbe4..ae728b2 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/IFeedManager.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IFeedManager.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.feed.lifecycle;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.util.concurrent.LinkedBlockingQueue;
 
@@ -56,4 +56,5 @@
      * @throws Exception
      */
     public void deliverMessage(FeedId feedId, IFeedMessage feedMessage) throws AsterixException;
+
 }
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/IFeedMessage.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IFeedMessage.java
similarity index 93%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/IFeedMessage.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IFeedMessage.java
index dcef2c8..85a15ab 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/feed/lifecycle/IFeedMessage.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IFeedMessage.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.feed.lifecycle;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.io.Serializable;
 
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/IGenericDatasetAdapterFactory.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IGenericDatasetAdapterFactory.java
similarity index 92%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/IGenericDatasetAdapterFactory.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IGenericDatasetAdapterFactory.java
index 0a178a7..c9d04cf 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/IGenericDatasetAdapterFactory.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IGenericDatasetAdapterFactory.java
@@ -12,11 +12,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.adapter.factory;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.util.Map;
 
-import edu.uci.ics.asterix.external.dataset.adapter.IDatasourceAdapter;
 import edu.uci.ics.asterix.om.types.IAType;
 
 /**
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/feed/managed/adapter/IManagedFeedAdapter.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IManagedFeedAdapter.java
similarity index 89%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/feed/managed/adapter/IManagedFeedAdapter.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IManagedFeedAdapter.java
index 87e8478..23d582d 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/feed/managed/adapter/IManagedFeedAdapter.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IManagedFeedAdapter.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.feed.managed.adapter;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.util.Map;
 
@@ -27,7 +27,7 @@
      * 
      * @throws Exception
      */
-    public void stop();
+    public void stop() throws Exception;
 
     /**
      * Modify the adapter configuration parameters. This method is called
@@ -38,6 +38,6 @@
      *            A HashMap containing the set of configuration parameters
      *            that need to be altered.
      */
-    public void alter(Map<String, String> properties);
+    public void alter(Map<String, Object> properties);
 
 }
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/ITypedDatasetAdapterFactory.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/ITypedDatasetAdapterFactory.java
similarity index 91%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/ITypedDatasetAdapterFactory.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/ITypedDatasetAdapterFactory.java
index 674bf52..7bfa24f 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/ITypedDatasetAdapterFactory.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/ITypedDatasetAdapterFactory.java
@@ -12,12 +12,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.adapter.factory;
+package edu.uci.ics.asterix.metadata.feeds;
 
 import java.util.Map;
 
-import edu.uci.ics.asterix.external.dataset.adapter.IDatasourceAdapter;
-
 /**
  * A base interface for an adapter factory that creates instance of an adapter kind that
  * is 'typed' in nature. A 'typed' adapter returns records with a pre-defined datatype.
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/ITypedDatasourceAdapter.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/ITypedDatasourceAdapter.java
similarity index 89%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/ITypedDatasourceAdapter.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/ITypedDatasourceAdapter.java
index 00b8530..08df1eb 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/ITypedDatasourceAdapter.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/ITypedDatasourceAdapter.java
@@ -12,8 +12,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.dataset.adapter;
+package edu.uci.ics.asterix.metadata.feeds;
 
+import edu.uci.ics.asterix.metadata.feeds.IDatasourceAdapter;
 import edu.uci.ics.asterix.om.types.ARecordType;
 
 /**
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/base/AMutableUnorderedList.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/base/AMutableUnorderedList.java
index fe0086b..7cb490c 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/base/AMutableUnorderedList.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/base/AMutableUnorderedList.java
@@ -20,15 +20,20 @@
 
 public final class AMutableUnorderedList extends AUnorderedList {
 
-    public AMutableUnorderedList(AUnorderedListType type) {
-        super(type);
-    }
+	public AMutableUnorderedList(AUnorderedListType type) {
+		super(type);
+	}
 
-    public AMutableUnorderedList(AUnorderedListType type, ArrayList<IAObject> sequence) {
-        super(type, sequence);
-    }
+	public AMutableUnorderedList(AUnorderedListType type,
+			ArrayList<IAObject> sequence) {
+		super(type, sequence);
+	}
 
-    public void add(IAObject obj) {
-        values.add(obj);
-    }
+	public void add(IAObject obj) {
+		values.add(obj);
+	}
+
+	public void clear() {
+		values.clear();
+	}
 }
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java
index 6cdaea0..1791a80 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java
@@ -29,82 +29,98 @@
 import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
 import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
 import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
+import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
 
 /*
  * Acts as an holder class for IndexRegistryProvider, AsterixStorageManager
  * instances that are accessed from the NCs. In addition an instance of ICCApplicationContext 
  * is stored for access by the CC.
  */
-public class AsterixAppContextInfo implements IAsterixApplicationContextInfo, IAsterixPropertiesProvider {
+public class AsterixAppContextInfo implements IAsterixApplicationContextInfo,
+		IAsterixPropertiesProvider {
 
-    private static AsterixAppContextInfo INSTANCE;
+	private static AsterixAppContextInfo INSTANCE;
 
-    private final ICCApplicationContext appCtx;
+	private final ICCApplicationContext appCtx;
 
-    private AsterixCompilerProperties compilerProperties;
-    private AsterixExternalProperties externalProperties;
-    private AsterixMetadataProperties metadataProperties;
-    private AsterixStorageProperties storageProperties;
-    private AsterixTransactionProperties txnProperties;
+	private AsterixCompilerProperties compilerProperties;
+	private AsterixExternalProperties externalProperties;
+	private AsterixMetadataProperties metadataProperties;
+	private AsterixStorageProperties storageProperties;
+	private AsterixTransactionProperties txnProperties;
+	private IHyracksClientConnection hcc;
 
-    public static void initialize(ICCApplicationContext ccAppCtx) throws AsterixException {
-        if (INSTANCE == null) {
-            INSTANCE = new AsterixAppContextInfo(ccAppCtx);
-        }
-        AsterixPropertiesAccessor propertiesAccessor = new AsterixPropertiesAccessor();
-        INSTANCE.compilerProperties = new AsterixCompilerProperties(propertiesAccessor);
-        INSTANCE.externalProperties = new AsterixExternalProperties(propertiesAccessor);
-        INSTANCE.metadataProperties = new AsterixMetadataProperties(propertiesAccessor);
-        INSTANCE.storageProperties = new AsterixStorageProperties(propertiesAccessor);
-        INSTANCE.txnProperties = new AsterixTransactionProperties(propertiesAccessor);
-        Logger.getLogger("edu.uci.ics").setLevel(INSTANCE.externalProperties.getLogLevel());
-    }
+	public static void initialize(ICCApplicationContext ccAppCtx,
+			IHyracksClientConnection hcc) throws AsterixException {
+		if (INSTANCE == null) {
+			INSTANCE = new AsterixAppContextInfo(ccAppCtx);
+		}
+		AsterixPropertiesAccessor propertiesAccessor = new AsterixPropertiesAccessor();
+		INSTANCE.compilerProperties = new AsterixCompilerProperties(
+				propertiesAccessor);
+		INSTANCE.externalProperties = new AsterixExternalProperties(
+				propertiesAccessor);
+		INSTANCE.metadataProperties = new AsterixMetadataProperties(
+				propertiesAccessor);
+		INSTANCE.storageProperties = new AsterixStorageProperties(
+				propertiesAccessor);
+		INSTANCE.txnProperties = new AsterixTransactionProperties(
+				propertiesAccessor);
+		INSTANCE.hcc = hcc;
+		Logger.getLogger("edu.uci.ics").setLevel(
+				INSTANCE.externalProperties.getLogLevel());
+	}
 
-    private AsterixAppContextInfo(ICCApplicationContext ccAppCtx) {
-        this.appCtx = ccAppCtx;
-    }
+	private AsterixAppContextInfo(ICCApplicationContext ccAppCtx) {
+		this.appCtx = ccAppCtx;
+	}
 
-    public static AsterixAppContextInfo getInstance() {
-        return INSTANCE;
-    }
+	public static AsterixAppContextInfo getInstance() {
+		return INSTANCE;
+	}
 
-    @Override
-    public ICCApplicationContext getCCApplicationContext() {
-        return appCtx;
-    }
+	@Override
+	public ICCApplicationContext getCCApplicationContext() {
+		return appCtx;
+	}
 
-    @Override
-    public AsterixStorageProperties getStorageProperties() {
-        return storageProperties;
-    }
+	@Override
+	public AsterixStorageProperties getStorageProperties() {
+		return storageProperties;
+	}
 
-    @Override
-    public AsterixTransactionProperties getTransactionProperties() {
-        return txnProperties;
-    }
+	@Override
+	public AsterixTransactionProperties getTransactionProperties() {
+		return txnProperties;
+	}
 
-    @Override
-    public AsterixCompilerProperties getCompilerProperties() {
-        return compilerProperties;
-    }
+	@Override
+	public AsterixCompilerProperties getCompilerProperties() {
+		return compilerProperties;
+	}
 
-    @Override
-    public AsterixMetadataProperties getMetadataProperties() {
-        return metadataProperties;
-    }
+	@Override
+	public AsterixMetadataProperties getMetadataProperties() {
+		return metadataProperties;
+	}
 
-    @Override
-    public AsterixExternalProperties getExternalProperties() {
-        return externalProperties;
-    }
+	@Override
+	public AsterixExternalProperties getExternalProperties() {
+		return externalProperties;
+	}
 
-    @Override
-    public IIndexLifecycleManagerProvider getIndexLifecycleManagerProvider() {
-        return AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER;
-    }
+	public IHyracksClientConnection getHcc() {
+		return hcc;
+	}
 
-    @Override
-    public IStorageManagerInterface getStorageManagerInterface() {
-        return AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER;
-    }
+	@Override
+	public IIndexLifecycleManagerProvider getIndexLifecycleManagerProvider() {
+		return AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER;
+	}
+
+	@Override
+	public IStorageManagerInterface getStorageManagerInterface() {
+		return AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER;
+	}
 }
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixClusterProperties.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixClusterProperties.java
index b9282b3..7daf04c 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixClusterProperties.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixClusterProperties.java
@@ -15,72 +15,92 @@
 package edu.uci.ics.asterix.om.util;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import edu.uci.ics.asterix.event.schema.cluster.Cluster;
+import edu.uci.ics.asterix.event.schema.cluster.Node;
+
 /**
  * A holder class for properties related to the Asterix cluster.
  */
 
 public class AsterixClusterProperties {
 
-    private static final Logger LOGGER = Logger.getLogger(AsterixClusterProperties.class.getName());
+	private static final Logger LOGGER = Logger
+			.getLogger(AsterixClusterProperties.class.getName());
 
-    private static final String IO_DEVICES = "iodevices";
+	private static final String IO_DEVICES = "iodevices";
 
-    public static final AsterixClusterProperties INSTANCE = new AsterixClusterProperties();
+	public static final AsterixClusterProperties INSTANCE = new AsterixClusterProperties();
 
-    private Map<String, Map<String, String>> ncConfiguration = new HashMap<String, Map<String, String>>();
+	private Map<String, Map<String, String>> ncConfiguration = new HashMap<String, Map<String, String>>();
 
-    private AsterixClusterProperties() {
-    }
+	private final Cluster cluster;
 
-    public enum State {
-        ACTIVE,
-        UNUSABLE
-    }
+	private AsterixClusterProperties() {
+		cluster = null;
+	}
 
-    private State state = State.UNUSABLE;
+	public enum State {
+		ACTIVE, UNUSABLE
+	}
 
-    public void removeNCConfiguration(String nodeId) {
-        state = State.UNUSABLE;
-        ncConfiguration.remove(nodeId);
-    }
+	private State state = State.UNUSABLE;
 
-    public void addNCConfiguration(String nodeId, Map<String, String> configuration) {
-        ncConfiguration.put(nodeId, configuration);
-        if (ncConfiguration.keySet().size() == AsterixAppContextInfo.getInstance().getMetadataProperties()
-                .getNodeNames().size()) {
-            state = State.ACTIVE;
-        }
-        if (LOGGER.isLoggable(Level.INFO)) {
-            LOGGER.info(" Registering configuration parameters for node id" + nodeId);
-        }
-    }
+	public void removeNCConfiguration(String nodeId) {
+		state = State.UNUSABLE;
+		ncConfiguration.remove(nodeId);
+	}
 
-    /**
-     * Returns the number of IO devices configured for a Node Controller
-     * 
-     * @param nodeId
-     *            unique identifier of the Node Controller
-     * @return number of IO devices. -1 if the node id is not valid. A node id is not valid
-     *         if it does not correspond to the set of registered Node Controllers.
-     */
-    public int getNumberOfIODevices(String nodeId) {
-        Map<String, String> ncConfig = ncConfiguration.get(nodeId);
-        if (ncConfig == null) {
-            if (LOGGER.isLoggable(Level.WARNING)) {
-                LOGGER.warning("Configuration parameters for nodeId" + nodeId
-                        + " not found. The node has not joined yet or has left.");
-            }
-            return -1;
-        }
-        return ncConfig.get(IO_DEVICES).split(",").length;
-    }
+	public void addNCConfiguration(String nodeId,
+			Map<String, String> configuration) {
+		ncConfiguration.put(nodeId, configuration);
+		if (ncConfiguration.keySet().size() == AsterixAppContextInfo
+				.getInstance().getMetadataProperties().getNodeNames().size()) {
+			state = State.ACTIVE;
+		}
+		if (LOGGER.isLoggable(Level.INFO)) {
+			LOGGER.info(" Registering configuration parameters for node id"
+					+ nodeId);
+		}
+	}
 
-    public State getState() {
-        return state;
-    }
+	/**
+	 * Returns the number of IO devices configured for a Node Controller
+	 * 
+	 * @param nodeId
+	 *            unique identifier of the Node Controller
+	 * @return number of IO devices. -1 if the node id is not valid. A node id
+	 *         is not valid if it does not correspond to the set of registered
+	 *         Node Controllers.
+	 */
+	public int getNumberOfIODevices(String nodeId) {
+		Map<String, String> ncConfig = ncConfiguration.get(nodeId);
+		if (ncConfig == null) {
+			if (LOGGER.isLoggable(Level.WARNING)) {
+				LOGGER.warning("Configuration parameters for nodeId"
+						+ nodeId
+						+ " not found. The node has not joined yet or has left.");
+			}
+			return -1;
+		}
+		return ncConfig.get(IO_DEVICES).split(",").length;
+	}
+
+	public State getState() {
+		return state;
+	}
+
+	public Cluster getCluster() {
+		return cluster;
+	}
+
+	public Node getAvailableSubstitutionNode() {
+		List<Node> subNodes = cluster.getSubstituteNodes().getNode();
+		return subNodes.isEmpty() ? null : subNodes.remove(0);
+	}
 
 }
diff --git a/asterix-test-framework/pom.xml b/asterix-test-framework/pom.xml
old mode 100755
new mode 100644
diff --git a/asterix-test-framework/src/main/resources/Catalog.xsd b/asterix-test-framework/src/main/resources/Catalog.xsd
old mode 100755
new mode 100644
diff --git a/asterix-tools/data/uscensus/dist.all.first.cleaned b/asterix-tools/data/uscensus/dist.all.first.cleaned
old mode 100755
new mode 100644
diff --git a/asterix-tools/data/uscensus/dist.all.last.cleaned b/asterix-tools/data/uscensus/dist.all.last.cleaned
old mode 100755
new mode 100644
diff --git a/asterix-tools/data/uscensus/dist.female.first.cleaned b/asterix-tools/data/uscensus/dist.female.first.cleaned
old mode 100755
new mode 100644
diff --git a/asterix-tools/data/uscensus/dist.male.first.cleaned b/asterix-tools/data/uscensus/dist.male.first.cleaned
old mode 100755
new mode 100644
diff --git a/asterix-tools/pom.xml b/asterix-tools/pom.xml
index 6c7db41..861dd79 100644
--- a/asterix-tools/pom.xml
+++ b/asterix-tools/pom.xml
@@ -148,6 +148,12 @@
 			<scope>compile</scope>
 		</dependency>
 		<dependency>
+			<groupId>edu.uci.ics.asterix</groupId>
+			<artifactId>asterix-external-data</artifactId>
+			<version>0.8.1-SNAPSHOT</version>
+			<scope>compile</scope>
+		</dependency>
+		<dependency>
 			<groupId>org.apache.httpcomponents</groupId>
 			<artifactId>httpclient</artifactId>
 			<version>4.2.2</version>
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/DataGenerator.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/DataGenerator.java
new file mode 100644
index 0000000..39537a8
--- /dev/null
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/DataGenerator.java
@@ -0,0 +1,2474 @@
+package edu.uci.ics.asterix.tools.external.data;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.CharBuffer;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class DataGenerator {
+
+    private static RandomDateGenerator randDateGen;
+    private static RandomNameGenerator randNameGen;
+    private static RandomEmploymentGenerator randEmpGen;
+    private static RandomMessageGenerator randMessageGen;
+    private static RandomLocationGenerator randLocationGen;
+
+    private static DistributionHandler fbDistHandler;
+    private static DistributionHandler twDistHandler;
+
+    private static int totalFbMessages;
+    private static int numFbOnlyUsers;
+    private static int totalTwMessages;
+    private static int numTwOnlyUsers;
+
+    private static int numCommonUsers;
+
+    private static int fbUserId;
+    private static int twUserId;
+
+    private static int fbMessageId;
+    private static int twMessageId;
+
+    private static Random random = new Random();
+
+    private static String commonUserFbSuffix = "_fb";
+    private static String commonUserTwSuffix = "_tw";
+
+    private static String outputDir;
+
+    private static PartitionConfiguration partition;
+
+    private static FacebookUser fbUser = new FacebookUser();
+    private static TwitterUser twUser = new TwitterUser();
+
+    private static FacebookMessage fbMessage = new FacebookMessage();
+    private static TweetMessage twMessage = new TweetMessage();
+
+    private static int duration;
+
+    private static DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
+
+    private static void generateFacebookOnlyUsers(int numFacebookUsers) throws IOException {
+        FileAppender appender = FileUtil.getFileAppender(outputDir + "/" + "fb_users.adm", true, true);
+        FileAppender messageAppender = FileUtil.getFileAppender(outputDir + "/" + "fb_message.adm", true, true);
+
+        for (int i = 0; i < numFacebookUsers; i++) {
+            getFacebookUser(null);
+            appender.appendToFile(fbUser.toString());
+            generateFacebookMessages(fbUser, messageAppender, -1);
+        }
+        appender.close();
+        messageAppender.close();
+    }
+
+    private static void generateTwitterOnlyUsers(int numTwitterUsers) throws IOException {
+        FileAppender appender = FileUtil.getFileAppender(outputDir + "/" + "tw_users.adm", true, true);
+        FileAppender messageAppender = FileUtil.getFileAppender(outputDir + "/" + "tw_message.adm", true, true);
+
+        for (int i = 0; i < numTwitterUsers; i++) {
+            getTwitterUser(null);
+            appender.appendToFile(twUser.toString());
+            generateTwitterMessages(twUser, messageAppender, -1);
+        }
+        appender.close();
+        messageAppender.close();
+    }
+
+    private static void generateCommonUsers() throws IOException {
+        FileAppender fbAppender = FileUtil.getFileAppender(outputDir + "/" + "fb_users.adm", true, false);
+        FileAppender twAppender = FileUtil.getFileAppender(outputDir + "/" + "tw_users.adm", true, false);
+        FileAppender fbMessageAppender = FileUtil.getFileAppender(outputDir + "/" + "fb_message.adm", true, false);
+        FileAppender twMessageAppender = FileUtil.getFileAppender(outputDir + "/" + "tw_message.adm", true, false);
+
+        for (int i = 0; i < numCommonUsers; i++) {
+            getFacebookUser(commonUserFbSuffix);
+            fbAppender.appendToFile(fbUser.toString());
+            generateFacebookMessages(fbUser, fbMessageAppender, -1);
+
+            getCorrespondingTwitterUser(fbUser);
+            twAppender.appendToFile(twUser.toString());
+            generateTwitterMessages(twUser, twMessageAppender, -1);
+        }
+
+        fbAppender.close();
+        twAppender.close();
+        fbMessageAppender.close();
+        twMessageAppender.close();
+    }
+
+    private static void generateFacebookMessages(FacebookUser user, FileAppender appender, int numMsg)
+            throws IOException {
+        Message message;
+        int numMessages = 0;
+        if (numMsg == -1) {
+            numMessages = fbDistHandler
+                    .getFromDistribution(fbUserId - partition.getTargetPartition().getFbUserKeyMin());
+        }
+        for (int i = 0; i < numMessages; i++) {
+            message = randMessageGen.getNextRandomMessage();
+            Point location = randLocationGen.getRandomPoint();
+            fbMessage.reset(fbMessageId++, user.getId(), random.nextInt(totalFbMessages + 1), location, message);
+            appender.appendToFile(fbMessage.toString());
+        }
+    }
+
+    private static void generateTwitterMessages(TwitterUser user, FileAppender appender, int numMsg) throws IOException {
+        Message message;
+        int numMessages = 0;
+        if (numMsg == -1) {
+            numMessages = twDistHandler
+                    .getFromDistribution(twUserId - partition.getTargetPartition().getTwUserKeyMin());
+        }
+
+        for (int i = 0; i < numMessages; i++) {
+            message = randMessageGen.getNextRandomMessage();
+            Point location = randLocationGen.getRandomPoint();
+            DateTime sendTime = randDateGen.getNextRandomDatetime();
+            twMessage.reset(twMessageId + "", user, location, sendTime, message.getReferredTopics(), message);
+            twMessageId++;
+            appender.appendToFile(twMessage.toString());
+        }
+    }
+
+    public static Iterator<TweetMessage> getTwitterMessageIterator() {
+        return new TweetMessageIterator(duration);
+    }
+
+    public static class TweetMessageIterator implements Iterator<TweetMessage> {
+
+        private final int duration;
+        private long startTime = 0;
+
+        public TweetMessageIterator(int duration) {
+            this.duration = duration;
+        }
+
+        @Override
+        public boolean hasNext() {
+            if (startTime == 0) {
+                startTime = System.currentTimeMillis();
+            }
+            return System.currentTimeMillis() - startTime < duration * 1000;
+        }
+
+        @Override
+        public TweetMessage next() {
+            getTwitterUser(null);
+            Message message = randMessageGen.getNextRandomMessage();
+            Point location = randLocationGen.getRandomPoint();
+            DateTime sendTime = randDateGen.getNextRandomDatetime();
+            twMessage.reset(twMessageId + "", twUser, location, sendTime, message.getReferredTopics(), message);
+            twMessageId++;
+            if (twUserId > numTwOnlyUsers) {
+                twUserId = 1;
+            }
+            return twMessage;
+
+        }
+
+        @Override
+        public void remove() {
+            // TODO Auto-generated method stub
+
+        }
+
+    }
+
+    public static class InitializationInfo {
+        public Date startDate = new Date(1, 1, 2005);
+        public Date endDate = new Date(8, 20, 2012);
+        public String[] lastNames = DataGenerator.lastNames;
+        public String[] firstNames = DataGenerator.firstNames;
+        public String[] vendors = DataGenerator.vendors;
+        public String[] jargon = DataGenerator.jargon;
+        public String[] org_list = DataGenerator.org_list;
+        public int percentEmployed = 90;
+        public Date employmentStartDate = new Date(1, 1, 2000);
+        public Date employmentEndDate = new Date(31, 12, 2012);
+        public int totalFbMessages;
+        public int numFbOnlyUsers;
+        public int totalTwMessages;
+        public int numTwOnlyUsers = 5000;
+        public int numCommonUsers;
+        public int fbUserIdMin;
+        public int fbMessageIdMin;
+        public int twUserIdMin;
+        public int twMessageIdMin;
+        public int timeDurationInSecs = 60;
+
+    }
+
+    public static void initialize(InitializationInfo info) {
+        randDateGen = new RandomDateGenerator(info.startDate, info.endDate);
+        randNameGen = new RandomNameGenerator(info.firstNames, info.lastNames);
+        randEmpGen = new RandomEmploymentGenerator(info.percentEmployed, info.employmentStartDate,
+                info.employmentEndDate, info.org_list);
+        randLocationGen = new RandomLocationGenerator(24, 49, 66, 98);
+        randMessageGen = new RandomMessageGenerator(info.vendors, info.jargon);
+        fbDistHandler = new DistributionHandler(info.totalFbMessages, 0.5, info.numFbOnlyUsers + info.numCommonUsers);
+        twDistHandler = new DistributionHandler(info.totalTwMessages, 0.5, info.numTwOnlyUsers + info.numCommonUsers);
+        fbUserId = info.fbUserIdMin;
+        twUserId = info.twUserIdMin;
+
+        fbMessageId = info.fbMessageIdMin;
+        twMessageId = info.fbMessageIdMin;
+        duration = info.timeDurationInSecs;
+    }
+
+    public static void main(String args[]) throws Exception {
+
+        String controllerInstallDir = null;
+        if (args.length < 2) {
+            printUsage();
+            System.exit(1);
+        } else {
+            controllerInstallDir = args[0];
+            String partitionConfXML = controllerInstallDir + "/output/partition-conf.xml";
+            String partitionName = args[1];
+            partition = XMLUtil.getPartitionConfiguration(partitionConfXML, partitionName);
+        }
+
+        // 1
+        randDateGen = new RandomDateGenerator(new Date(1, 1, 2005), new Date(8, 20, 2012));
+
+        String firstNameFile = controllerInstallDir + "/metadata/firstNames.txt";
+        String lastNameFile = controllerInstallDir + "/metadata/lastNames.txt";
+        String vendorFile = controllerInstallDir + "/metadata/vendors.txt";
+        String jargonFile = controllerInstallDir + "/metadata/jargon.txt";
+        String orgList = controllerInstallDir + "/metadata/org_list.txt";
+
+        randNameGen = new RandomNameGenerator(firstNameFile, lastNameFile);
+        randEmpGen = new RandomEmploymentGenerator(90, new Date(1, 1, 2000), new Date(8, 20, 2012), orgList);
+        randLocationGen = new RandomLocationGenerator(24, 49, 66, 98);
+        randMessageGen = new RandomMessageGenerator(vendorFile, jargonFile);
+
+        totalFbMessages = partition.getTargetPartition().getFbMessageIdMax()
+                - partition.getTargetPartition().getFbMessageIdMin() + 1;
+        numFbOnlyUsers = (partition.getTargetPartition().getFbUserKeyMax()
+                - partition.getTargetPartition().getFbUserKeyMin() + 1)
+                - partition.getTargetPartition().getCommonUsers();
+
+        totalTwMessages = partition.getTargetPartition().getTwMessageIdMax()
+                - partition.getTargetPartition().getTwMessageIdMin() + 1;
+        numTwOnlyUsers = (partition.getTargetPartition().getTwUserKeyMax()
+                - partition.getTargetPartition().getTwUserKeyMin() + 1)
+                - partition.getTargetPartition().getCommonUsers();
+
+        numCommonUsers = partition.getTargetPartition().getCommonUsers();
+        fbDistHandler = new DistributionHandler(totalFbMessages, 0.5, numFbOnlyUsers + numCommonUsers);
+        twDistHandler = new DistributionHandler(totalTwMessages, 0.5, numTwOnlyUsers + numCommonUsers);
+
+        fbUserId = partition.getTargetPartition().getFbUserKeyMin();
+        twUserId = partition.getTargetPartition().getTwUserKeyMin();
+
+        fbMessageId = partition.getTargetPartition().getFbMessageIdMin();
+        twMessageId = partition.getTargetPartition().getTwMessageIdMin();
+
+        outputDir = partition.getSourcePartition().getPath();
+        generateData();
+    }
+
+    public static void printUsage() {
+        System.out.println(" Error: Invalid number of arguments ");
+        System.out.println(" Usage :" + " DataGenerator <path to configuration file> <partition name> ");
+    }
+
+    public static void generateData() throws IOException {
+        generateFacebookOnlyUsers(numFbOnlyUsers);
+        generateTwitterOnlyUsers(numTwOnlyUsers);
+        generateCommonUsers();
+        System.out.println("Partition :" + partition.getTargetPartition().getName() + " finished");
+    }
+
+    public static void getFacebookUser(String usernameSuffix) {
+        String suggestedName = randNameGen.getRandomName();
+        String[] nameComponents = suggestedName.split(" ");
+        String name = nameComponents[0] + nameComponents[1];
+        if (usernameSuffix != null) {
+            name = name + usernameSuffix;
+        }
+        String alias = nameComponents[0];
+        String userSince = randDateGen.getNextRandomDatetime().toString();
+        int numFriends = random.nextInt(25);
+        int[] friendIds = RandomUtil.getKFromN(numFriends, (numFbOnlyUsers + numCommonUsers));
+        Employment emp = randEmpGen.getRandomEmployment();
+        fbUser.reset(fbUserId++, alias, name, userSince, friendIds, emp);
+    }
+
+    public static void getTwitterUser(String usernameSuffix) {
+        String suggestedName = randNameGen.getRandomName();
+        String[] nameComponents = suggestedName.split(" ");
+        String screenName = nameComponents[0] + nameComponents[1] + randNameGen.getRandomNameSuffix();
+        String name = suggestedName;
+        if (usernameSuffix != null) {
+            name = name + usernameSuffix;
+        }
+        int numFriends = random.nextInt((int) (100)); // draw from Zipfian
+        int statusesCount = random.nextInt(500); // draw from Zipfian
+        int followersCount = random.nextInt((int) (200));
+        twUser.reset(screenName, numFriends, statusesCount, name, followersCount);
+        twUserId++;
+    }
+
+    public static void getCorrespondingTwitterUser(FacebookUser fbUser) {
+        String screenName = fbUser.getName().substring(0, fbUser.getName().lastIndexOf(commonUserFbSuffix))
+                + commonUserTwSuffix;
+        String name = screenName.split(" ")[0];
+        int numFriends = random.nextInt((int) ((numTwOnlyUsers + numCommonUsers)));
+        int statusesCount = random.nextInt(500); // draw from Zipfian
+        int followersCount = random.nextInt((int) (numTwOnlyUsers + numCommonUsers));
+        twUser.reset(screenName, numFriends, statusesCount, name, followersCount);
+    }
+
+    public static class RandomDateGenerator {
+
+        private final Date startDate;
+        private final Date endDate;
+        private final Random random = new Random();
+        private final int yearDifference;
+        private Date workingDate;
+        private Date recentDate;
+        private DateTime dateTime;
+
+        public RandomDateGenerator(Date startDate, Date endDate) {
+            this.startDate = startDate;
+            this.endDate = endDate;
+            yearDifference = endDate.getYear() - startDate.getYear() + 1;
+            workingDate = new Date();
+            recentDate = new Date();
+            dateTime = new DateTime();
+        }
+
+        public Date getStartDate() {
+            return startDate;
+        }
+
+        public Date getEndDate() {
+            return endDate;
+        }
+
+        public Date getNextRandomDate() {
+            int year = random.nextInt(yearDifference) + startDate.getYear();
+            int month;
+            int day;
+            if (year == endDate.getYear()) {
+                month = random.nextInt(endDate.getMonth()) + 1;
+                if (month == endDate.getMonth()) {
+                    day = random.nextInt(endDate.getDay()) + 1;
+                } else {
+                    day = random.nextInt(28) + 1;
+                }
+            } else {
+                month = random.nextInt(12) + 1;
+                day = random.nextInt(28) + 1;
+            }
+            workingDate.reset(month, day, year);
+            return workingDate;
+        }
+
+        public DateTime getNextRandomDatetime() {
+            Date randomDate = getNextRandomDate();
+            dateTime.reset(randomDate);
+            return dateTime;
+        }
+
+        public Date getNextRecentDate(Date date) {
+            int year = date.getYear()
+                    + (date.getYear() == endDate.getYear() ? 0 : random.nextInt(endDate.getYear() - date.getYear()));
+            int month = (year == endDate.getYear()) ? date.getMonth() == endDate.getMonth() ? (endDate.getMonth())
+                    : (date.getMonth() + random.nextInt(endDate.getMonth() - date.getMonth())) : random.nextInt(12) + 1;
+
+            int day = (year == endDate.getYear()) ? month == endDate.getMonth() ? date.getDay() == endDate.getDay() ? endDate
+                    .getDay() : date.getDay() + random.nextInt(endDate.getDay() - date.getDay())
+                    : random.nextInt(28) + 1
+                    : random.nextInt(28) + 1;
+            recentDate.reset(month, day, year);
+            return recentDate;
+        }
+
+        public static void main(String args[]) throws Exception {
+            Date date = new Date(2, 20, 2012);
+            RandomDateGenerator dgen = new RandomDateGenerator(new Date(1, 1, 2005), new Date(8, 20, 2012));
+            while (true) {
+                Date nextDate = dgen.getNextRandomDate();
+                if (nextDate.getDay() == 0) {
+                    throw new Exception("invalid date " + nextDate);
+                }
+
+                // System.out.println(" original date: " + date);
+                System.out.println(nextDate);
+            }
+        }
+    }
+
+    public static class DateTime extends Date {
+
+        private String hour = "10";
+        private String min = "10";
+        private String sec = "00";
+        private long chrononTime;
+
+        public DateTime(int month, int day, int year, String hour, String min, String sec) {
+            super(month, day, year);
+            this.hour = hour;
+            this.min = min;
+            this.sec = sec;
+            chrononTime = new java.util.Date(year, month, day, Integer.parseInt(hour), Integer.parseInt(min),
+                    Integer.parseInt(sec)).getTime();
+        }
+
+        public void reset(int month, int day, int year, String hour, String min, String sec) {
+            super.setDay(month);
+            super.setDay(day);
+            super.setYear(year);
+            this.hour = hour;
+            this.min = min;
+            this.sec = sec;
+            chrononTime = new java.util.Date(year, month, day, Integer.parseInt(hour), Integer.parseInt(min),
+                    Integer.parseInt(sec)).getTime();
+        }
+
+        public DateTime() {
+        }
+
+        public DateTime(Date date) {
+            super(date.getMonth(), date.getDay(), date.getYear());
+        }
+
+        public void reset(Date date) {
+            reset(date.getMonth(), date.getDay(), date.getYear());
+        }
+
+        public DateTime(Date date, int hour, int min, int sec) {
+            super(date.getMonth(), date.getDay(), date.getYear());
+            this.hour = (hour < 10) ? "0" : "" + hour;
+            this.min = (min < 10) ? "0" : "" + min;
+            this.sec = (sec < 10) ? "0" : "" + sec;
+        }
+
+        public long getChrononTime() {
+            return chrononTime;
+        }
+
+        public String getHour() {
+            return hour;
+        }
+
+        public String getMin() {
+            return min;
+        }
+
+        public String getSec() {
+            return sec;
+        }
+
+        public String toString() {
+            StringBuilder builder = new StringBuilder();
+            builder.append("datetime");
+            builder.append("(\"");
+            builder.append(super.getYear());
+            builder.append("-");
+            builder.append(super.getMonth() < 10 ? "0" + super.getMonth() : super.getMonth());
+            builder.append("-");
+            builder.append(super.getDay() < 10 ? "0" + super.getDay() : super.getDay());
+            builder.append("T");
+            builder.append(hour + ":" + min + ":" + sec);
+            builder.append("\")");
+            return builder.toString();
+        }
+    }
+
+    public static class Message {
+
+        private char[] message = new char[500];
+        private List<String> referredTopics;
+        private int length;
+
+        public Message(char[] m, List<String> referredTopics) {
+            System.arraycopy(m, 0, message, 0, m.length);
+            length = m.length;
+            this.referredTopics = referredTopics;
+        }
+
+        public Message() {
+            referredTopics = new ArrayList<String>();
+            length = 0;
+        }
+
+        public char[] getMessage() {
+            return message;
+        }
+
+        public List<String> getReferredTopics() {
+            return referredTopics;
+        }
+
+        public void reset(char[] m, int offset, int length, List<String> referredTopics) {
+            System.arraycopy(m, offset, message, 0, length);
+            this.length = length;
+            this.referredTopics = referredTopics;
+        }
+
+        public int getLength() {
+            return length;
+        }
+
+        public char charAt(int index) {
+            return message[index];
+        }
+
+    }
+
+    public static class Point {
+
+        private float latitude;
+        private float longitude;
+
+        public float getLatitude() {
+            return latitude;
+        }
+
+        public float getLongitude() {
+            return longitude;
+        }
+
+        public Point(float latitude, float longitude) {
+            this.latitude = latitude;
+            this.longitude = longitude;
+        }
+
+        public void reset(float latitude, float longitude) {
+            this.latitude = latitude;
+            this.longitude = longitude;
+        }
+
+        public Point() {
+        }
+
+        public String toString() {
+            StringBuilder builder = new StringBuilder();
+            builder.append("point(\"" + latitude + "," + longitude + "\")");
+            return builder.toString();
+        }
+    }
+
+    public static class RandomNameGenerator {
+
+        private String[] firstNames;
+        private String[] lastNames;
+
+        private final Random random = new Random();
+
+        private final String[] connectors = new String[] { "_", "#", "$", "@" };
+
+        public RandomNameGenerator(String firstNameFilePath, String lastNameFilePath) throws IOException {
+            firstNames = FileUtil.listyFile(new File(firstNameFilePath)).toArray(new String[] {});
+            lastNames = FileUtil.listyFile(new File(lastNameFilePath)).toArray(new String[] {});
+        }
+
+        public RandomNameGenerator(String[] firstNames, String[] lastNames) {
+            this.firstNames = firstNames;
+            this.lastNames = lastNames;
+        }
+
+        public String getRandomName() {
+            String name;
+            name = getSuggestedName();
+            return name;
+
+        }
+
+        private String getSuggestedName() {
+            int firstNameIndex = random.nextInt(firstNames.length);
+            int lastNameIndex = random.nextInt(lastNames.length);
+            String suggestedName = firstNames[firstNameIndex] + " " + lastNames[lastNameIndex];
+            return suggestedName;
+        }
+
+        public String getRandomNameSuffix() {
+            return connectors[random.nextInt(connectors.length)] + random.nextInt(1000);
+        }
+    }
+
+    public static class RandomMessageGenerator {
+
+        private final MessageTemplate messageTemplate;
+
+        public RandomMessageGenerator(String vendorFilePath, String jargonFilePath) throws IOException {
+            List<String> vendors = FileUtil.listyFile(new File(vendorFilePath));
+            List<String> jargon = FileUtil.listyFile(new File(jargonFilePath));
+            this.messageTemplate = new MessageTemplate(vendors, jargon);
+        }
+
+        public RandomMessageGenerator(String[] vendors, String[] jargon) {
+            List<String> vendorList = new ArrayList<String>();
+            for (String v : vendors) {
+                vendorList.add(v);
+            }
+            List<String> jargonList = new ArrayList<String>();
+            for (String j : jargon) {
+                jargonList.add(j);
+            }
+            this.messageTemplate = new MessageTemplate(vendorList, jargonList);
+        }
+
+        public Message getNextRandomMessage() {
+            return messageTemplate.getNextMessage();
+        }
+    }
+
+    public static class AbstractMessageTemplate {
+
+        protected final Random random = new Random();
+
+        protected String[] positiveVerbs = new String[] { "like", "love" };
+        protected String[] negativeVerbs = new String[] { "dislike", "hate", "can't stand" };
+
+        protected String[] negativeAdjectives = new String[] { "horrible", "bad", "terrible", "OMG" };
+        protected String[] postiveAdjectives = new String[] { "good", "awesome", "amazing", "mind-blowing" };
+
+        protected String[] otherWords = new String[] { "the", "its" };
+    }
+
+    public static class MessageTemplate extends AbstractMessageTemplate {
+
+        private List<String> vendors;
+        private List<String> jargon;
+        private CharBuffer buffer;
+        private List<String> referredTopics;
+        private Message message = new Message();
+
+        public MessageTemplate(List<String> vendors, List<String> jargon) {
+            this.vendors = vendors;
+            this.jargon = jargon;
+            buffer = CharBuffer.allocate(2500);
+            referredTopics = new ArrayList<String>();
+        }
+
+        public Message getNextMessage() {
+            buffer.position(0);
+            buffer.limit(2500);
+            referredTopics.clear();
+            boolean isPositive = random.nextBoolean();
+            String[] verbArray = isPositive ? positiveVerbs : negativeVerbs;
+            String[] adjectiveArray = isPositive ? postiveAdjectives : negativeAdjectives;
+            String verb = verbArray[random.nextInt(verbArray.length)];
+            String adjective = adjectiveArray[random.nextInt(adjectiveArray.length)];
+
+            buffer.put(" ");
+            buffer.put(verb);
+            buffer.put(" ");
+            String vendor = vendors.get(random.nextInt(vendors.size()));
+            referredTopics.add(vendor);
+            buffer.append(vendor);
+            buffer.append(" ");
+            buffer.append(otherWords[random.nextInt(otherWords.length)]);
+            buffer.append(" ");
+            String jargonTerm = jargon.get(random.nextInt(jargon.size()));
+            referredTopics.add(jargonTerm);
+            buffer.append(jargonTerm);
+            buffer.append(" is ");
+            buffer.append(adjective);
+            if (random.nextBoolean()) {
+                buffer.append(isPositive ? ":)" : ":(");
+            }
+
+            buffer.flip();
+            message.reset(buffer.array(), 0, buffer.limit(), referredTopics);
+            return message;
+        }
+    }
+
+    public static class RandomUtil {
+
+        public static Random random = new Random();
+
+        public static int[] getKFromN(int k, int n) {
+            int[] result = new int[k];
+            int cnt = 0;
+            HashSet<Integer> values = new HashSet<Integer>();
+            while (cnt < k) {
+                int val = random.nextInt(n + 1);
+                if (values.contains(val)) {
+                    continue;
+                }
+
+                result[cnt++] = val;
+                values.add(val);
+            }
+            return result;
+        }
+    }
+
+    public static class FileUtil {
+
+        public static List<String> listyFile(File file) throws IOException {
+
+            BufferedReader reader = new BufferedReader(new FileReader(file));
+            String line;
+            List<String> list = new ArrayList<String>();
+            while (true) {
+                line = reader.readLine();
+                if (line == null) {
+                    break;
+                }
+                list.add(line);
+            }
+            return list;
+        }
+
+        public static FileAppender getFileAppender(String filePath, boolean createIfNotExists, boolean overwrite)
+                throws IOException {
+            return new FileAppender(filePath, createIfNotExists, overwrite);
+        }
+    }
+
+    public static class FileAppender {
+
+        private final BufferedWriter writer;
+
+        public FileAppender(String filePath, boolean createIfNotExists, boolean overwrite) throws IOException {
+            File file = new File(filePath);
+            if (!file.exists()) {
+                if (createIfNotExists) {
+                    new File(file.getParent()).mkdirs();
+                } else {
+                    throw new IOException("path " + filePath + " does not exists");
+                }
+            }
+            this.writer = new BufferedWriter(new FileWriter(file, !overwrite));
+        }
+
+        public void appendToFile(String content) throws IOException {
+            writer.append(content);
+            writer.append("\n");
+        }
+
+        public void close() throws IOException {
+            writer.close();
+        }
+    }
+
+    public static class RandomEmploymentGenerator {
+
+        private final int percentEmployed;
+        private final Random random = new Random();
+        private final RandomDateGenerator randDateGen;
+        private final List<String> organizations;
+        private Employment emp;
+
+        public RandomEmploymentGenerator(int percentEmployed, Date beginEmpDate, Date endEmpDate, String orgListPath)
+                throws IOException {
+            this.percentEmployed = percentEmployed;
+            this.randDateGen = new RandomDateGenerator(beginEmpDate, endEmpDate);
+            organizations = FileUtil.listyFile(new File(orgListPath));
+            emp = new Employment();
+        }
+
+        public RandomEmploymentGenerator(int percentEmployed, Date beginEmpDate, Date endEmpDate, String[] orgList) {
+            this.percentEmployed = percentEmployed;
+            this.randDateGen = new RandomDateGenerator(beginEmpDate, endEmpDate);
+            organizations = new ArrayList<String>();
+            for (String org : orgList) {
+                organizations.add(org);
+            }
+            emp = new Employment();
+        }
+
+        public Employment getRandomEmployment() {
+            int empployed = random.nextInt(100) + 1;
+            boolean isEmployed = false;
+            if (empployed <= percentEmployed) {
+                isEmployed = true;
+            }
+            Date startDate = randDateGen.getNextRandomDate();
+            Date endDate = null;
+            if (!isEmployed) {
+                endDate = randDateGen.getNextRecentDate(startDate);
+            }
+            String org = organizations.get(random.nextInt(organizations.size()));
+            emp.reset(org, startDate, endDate);
+            return emp;
+        }
+    }
+
+    public static class RandomLocationGenerator {
+
+        private Random random = new Random();
+
+        private final int beginLat;
+        private final int endLat;
+        private final int beginLong;
+        private final int endLong;
+
+        private Point point;
+
+        public RandomLocationGenerator(int beginLat, int endLat, int beginLong, int endLong) {
+            this.beginLat = beginLat;
+            this.endLat = endLat;
+            this.beginLong = beginLong;
+            this.endLong = endLong;
+            this.point = new Point();
+        }
+
+        public Point getRandomPoint() {
+            int latMajor = beginLat + random.nextInt(endLat - beginLat);
+            int latMinor = random.nextInt(100);
+            float latitude = latMajor + ((float) latMinor) / 100;
+
+            int longMajor = beginLong + random.nextInt(endLong - beginLong);
+            int longMinor = random.nextInt(100);
+            float longitude = longMajor + ((float) longMinor) / 100;
+
+            point.reset(latitude, longitude);
+            return point;
+        }
+
+    }
+
+    public static class PartitionConfiguration {
+
+        private final TargetPartition targetPartition;
+        private final SourcePartition sourcePartition;
+
+        public PartitionConfiguration(SourcePartition sourcePartition, TargetPartition targetPartition) {
+            this.sourcePartition = sourcePartition;
+            this.targetPartition = targetPartition;
+        }
+
+        public TargetPartition getTargetPartition() {
+            return targetPartition;
+        }
+
+        public SourcePartition getSourcePartition() {
+            return sourcePartition;
+        }
+
+    }
+
+    public static class SourcePartition {
+
+        private final String name;
+        private final String host;
+        private final String path;
+
+        public SourcePartition(String name, String host, String path) {
+            this.name = name;
+            this.host = host;
+            this.path = path;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public String getHost() {
+            return host;
+        }
+
+        public String getPath() {
+            return path;
+        }
+    }
+
+    public static class TargetPartition {
+        private final String name;
+        private final String host;
+        private final String path;
+        private final int fbUserKeyMin;
+        private final int fbUserKeyMax;
+        private final int twUserKeyMin;
+        private final int twUserKeyMax;
+        private final int fbMessageIdMin;
+        private final int fbMessageIdMax;
+        private final int twMessageIdMin;
+        private final int twMessageIdMax;
+        private final int commonUsers;
+
+        public TargetPartition(String partitionName, String host, String path, int fbUserKeyMin, int fbUserKeyMax,
+                int twUserKeyMin, int twUserKeyMax, int fbMessageIdMin, int fbMessageIdMax, int twMessageIdMin,
+                int twMessageIdMax, int commonUsers) {
+            this.name = partitionName;
+            this.host = host;
+            this.path = path;
+            this.fbUserKeyMin = fbUserKeyMin;
+            this.fbUserKeyMax = fbUserKeyMax;
+            this.twUserKeyMin = twUserKeyMin;
+            this.twUserKeyMax = twUserKeyMax;
+            this.twMessageIdMin = twMessageIdMin;
+            this.twMessageIdMax = twMessageIdMax;
+            this.fbMessageIdMin = fbMessageIdMin;
+            this.fbMessageIdMax = fbMessageIdMax;
+            this.commonUsers = commonUsers;
+        }
+
+        public String toString() {
+            StringBuilder builder = new StringBuilder();
+            builder.append(name);
+            builder.append(" ");
+            builder.append(host);
+            builder.append("\n");
+            builder.append(path);
+            builder.append("\n");
+            builder.append("fbUser:key:min");
+            builder.append(fbUserKeyMin);
+
+            builder.append("\n");
+            builder.append("fbUser:key:max");
+            builder.append(fbUserKeyMax);
+
+            builder.append("\n");
+            builder.append("twUser:key:min");
+            builder.append(twUserKeyMin);
+
+            builder.append("\n");
+            builder.append("twUser:key:max");
+            builder.append(twUserKeyMax);
+
+            builder.append("\n");
+            builder.append("fbMessage:key:min");
+            builder.append(fbMessageIdMin);
+
+            builder.append("\n");
+            builder.append("fbMessage:key:max");
+            builder.append(fbMessageIdMax);
+
+            builder.append("\n");
+            builder.append("twMessage:key:min");
+            builder.append(twMessageIdMin);
+
+            builder.append("\n");
+            builder.append("twMessage:key:max");
+            builder.append(twMessageIdMax);
+
+            builder.append("\n");
+            builder.append("twMessage:key:max");
+            builder.append(twMessageIdMax);
+
+            builder.append("\n");
+            builder.append("commonUsers");
+            builder.append(commonUsers);
+
+            return new String(builder);
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public String getHost() {
+            return host;
+        }
+
+        public int getFbUserKeyMin() {
+            return fbUserKeyMin;
+        }
+
+        public int getFbUserKeyMax() {
+            return fbUserKeyMax;
+        }
+
+        public int getTwUserKeyMin() {
+            return twUserKeyMin;
+        }
+
+        public int getTwUserKeyMax() {
+            return twUserKeyMax;
+        }
+
+        public int getFbMessageIdMin() {
+            return fbMessageIdMin;
+        }
+
+        public int getFbMessageIdMax() {
+            return fbMessageIdMax;
+        }
+
+        public int getTwMessageIdMin() {
+            return twMessageIdMin;
+        }
+
+        public int getTwMessageIdMax() {
+            return twMessageIdMax;
+        }
+
+        public int getCommonUsers() {
+            return commonUsers;
+        }
+
+        public String getPath() {
+            return path;
+        }
+    }
+
+    public static class Employment {
+
+        private String organization;
+        private Date startDate;
+        private Date endDate;
+
+        public Employment(String organization, Date startDate, Date endDate) {
+            this.organization = organization;
+            this.startDate = startDate;
+            this.endDate = endDate;
+        }
+
+        public Employment() {
+        }
+
+        public void reset(String organization, Date startDate, Date endDate) {
+            this.organization = organization;
+            this.startDate = startDate;
+            this.endDate = endDate;
+        }
+
+        public String getOrganization() {
+            return organization;
+        }
+
+        public Date getStartDate() {
+            return startDate;
+        }
+
+        public Date getEndDate() {
+            return endDate;
+        }
+
+        public String toString() {
+            StringBuilder builder = new StringBuilder("");
+            builder.append("{");
+            builder.append("\"organization-name\":");
+            builder.append("\"" + organization + "\"");
+            builder.append(",");
+            builder.append("\"start-date\":");
+            builder.append(startDate);
+            if (endDate != null) {
+                builder.append(",");
+                builder.append("\"end-date\":");
+                builder.append(endDate);
+            }
+            builder.append("}");
+            return new String(builder);
+        }
+
+    }
+
+    public static class FacebookMessage {
+
+        private int messageId;
+        private int authorId;
+        private int inResponseTo;
+        private Point senderLocation;
+        private Message message;
+
+        public int getMessageId() {
+            return messageId;
+        }
+
+        public int getAuthorID() {
+            return authorId;
+        }
+
+        public Point getSenderLocation() {
+            return senderLocation;
+        }
+
+        public Message getMessage() {
+            return message;
+        }
+
+        public int getInResponseTo() {
+            return inResponseTo;
+        }
+
+        public FacebookMessage() {
+
+        }
+
+        public FacebookMessage(int messageId, int authorId, int inResponseTo, Point senderLocation, Message message) {
+            this.messageId = messageId;
+            this.authorId = authorId;
+            this.inResponseTo = inResponseTo;
+            this.senderLocation = senderLocation;
+            this.message = message;
+        }
+
+        public void reset(int messageId, int authorId, int inResponseTo, Point senderLocation, Message message) {
+            this.messageId = messageId;
+            this.authorId = authorId;
+            this.inResponseTo = inResponseTo;
+            this.senderLocation = senderLocation;
+            this.message = message;
+        }
+
+        public String toString() {
+            StringBuilder builder = new StringBuilder();
+            builder.append("{");
+            builder.append("\"message-id\":");
+            builder.append(messageId);
+            builder.append(",");
+            builder.append("\"author-id\":");
+            builder.append(authorId);
+            builder.append(",");
+            builder.append("\"in-response-to\":");
+            builder.append(inResponseTo);
+            builder.append(",");
+            builder.append("\"sender-location\":");
+            builder.append(senderLocation);
+            builder.append(",");
+            builder.append("\"message\":");
+            builder.append("\"");
+            for (int i = 0; i < message.getLength(); i++) {
+                builder.append(message.charAt(i));
+            }
+            builder.append("\"");
+            builder.append("}");
+            return new String(builder);
+        }
+    }
+
+    public static class FacebookUser {
+
+        private int id;
+        private String alias;
+        private String name;
+        private String userSince;
+        private int[] friendIds;
+        private Employment employment;
+
+        public FacebookUser() {
+
+        }
+
+        public FacebookUser(int id, String alias, String name, String userSince, int[] friendIds, Employment employment) {
+            this.id = id;
+            this.alias = alias;
+            this.name = name;
+            this.userSince = userSince;
+            this.friendIds = friendIds;
+            this.employment = employment;
+        }
+
+        public int getId() {
+            return id;
+        }
+
+        public String getAlias() {
+            return alias;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public String getUserSince() {
+            return userSince;
+        }
+
+        public int[] getFriendIds() {
+            return friendIds;
+        }
+
+        public Employment getEmployment() {
+            return employment;
+        }
+
+        public String toString() {
+            StringBuilder builder = new StringBuilder();
+            builder.append("{");
+            builder.append("\"id\":" + id);
+            builder.append(",");
+            builder.append("\"alias\":" + "\"" + alias + "\"");
+            builder.append(",");
+            builder.append("\"name\":" + "\"" + name + "\"");
+            builder.append(",");
+            builder.append("\"user-since\":" + userSince);
+            builder.append(",");
+            builder.append("\"friend-ids\":");
+            builder.append("{{");
+            for (int i = 0; i < friendIds.length; i++) {
+                builder.append(friendIds[i]);
+                builder.append(",");
+            }
+            if (friendIds.length > 0) {
+                builder.deleteCharAt(builder.lastIndexOf(","));
+            }
+            builder.append("}}");
+            builder.append(",");
+            builder.append("\"employment\":");
+            builder.append("[");
+            builder.append(employment.toString());
+            builder.append("]");
+            builder.append("}");
+            return builder.toString();
+        }
+
+        public void setId(int id) {
+            this.id = id;
+        }
+
+        public void setAlias(String alias) {
+            this.alias = alias;
+        }
+
+        public void setName(String name) {
+            this.name = name;
+        }
+
+        public void setUserSince(String userSince) {
+            this.userSince = userSince;
+        }
+
+        public void setFriendIds(int[] friendIds) {
+            this.friendIds = friendIds;
+        }
+
+        public void setEmployment(Employment employment) {
+            this.employment = employment;
+        }
+
+        public void reset(int id, String alias, String name, String userSince, int[] friendIds, Employment employment) {
+            this.id = id;
+            this.alias = alias;
+            this.name = name;
+            this.userSince = userSince;
+            this.friendIds = friendIds;
+            this.employment = employment;
+        }
+    }
+
+    public static class TweetMessage {
+
+        private String tweetid;
+        private TwitterUser user;
+        private Point senderLocation;
+        private DateTime sendTime;
+        private List<String> referredTopics;
+        private Message messageText;
+
+        public TweetMessage() {
+
+        }
+
+        public TweetMessage(String tweetid, TwitterUser user, Point senderLocation, DateTime sendTime,
+                List<String> referredTopics, Message messageText) {
+            this.tweetid = tweetid;
+            this.user = user;
+            this.senderLocation = senderLocation;
+            this.sendTime = sendTime;
+            this.referredTopics = referredTopics;
+            this.messageText = messageText;
+        }
+
+        public void reset(String tweetid, TwitterUser user, Point senderLocation, DateTime sendTime,
+                List<String> referredTopics, Message messageText) {
+            this.tweetid = tweetid;
+            this.user = user;
+            this.senderLocation = senderLocation;
+            this.sendTime = sendTime;
+            this.referredTopics = referredTopics;
+            this.messageText = messageText;
+        }
+
+        public String toString() {
+            StringBuilder builder = new StringBuilder();
+            builder.append("{");
+            builder.append("\"tweetid\":");
+            builder.append("\"" + tweetid + "\"");
+            builder.append(",");
+            builder.append("\"user\":");
+            builder.append(user);
+            builder.append(",");
+            builder.append("\"sender-location\":");
+            builder.append(senderLocation);
+            builder.append(",");
+            builder.append("\"send-time\":");
+            builder.append(sendTime);
+            builder.append(",");
+            builder.append("\"referred-topics\":");
+            builder.append("{{");
+            for (String topic : referredTopics) {
+                builder.append("\"" + topic + "\"");
+                builder.append(",");
+            }
+            if (referredTopics.size() > 0) {
+                builder.deleteCharAt(builder.lastIndexOf(","));
+            }
+            builder.append("}}");
+            builder.append(",");
+            builder.append("\"message-text\":");
+            builder.append("\"");
+            for (int i = 0; i < messageText.getLength(); i++) {
+                builder.append(messageText.charAt(i));
+            }
+            builder.append("\"");
+            builder.append("}");
+            return new String(builder);
+        }
+
+        public String getTweetid() {
+            return tweetid;
+        }
+
+        public void setTweetid(String tweetid) {
+            this.tweetid = tweetid;
+        }
+
+        public TwitterUser getUser() {
+            return user;
+        }
+
+        public void setUser(TwitterUser user) {
+            this.user = user;
+        }
+
+        public Point getSenderLocation() {
+            return senderLocation;
+        }
+
+        public void setSenderLocation(Point senderLocation) {
+            this.senderLocation = senderLocation;
+        }
+
+        public DateTime getSendTime() {
+            return sendTime;
+        }
+
+        public void setSendTime(DateTime sendTime) {
+            this.sendTime = sendTime;
+        }
+
+        public List<String> getReferredTopics() {
+            return referredTopics;
+        }
+
+        public void setReferredTopics(List<String> referredTopics) {
+            this.referredTopics = referredTopics;
+        }
+
+        public Message getMessageText() {
+            return messageText;
+        }
+
+        public void setMessageText(Message messageText) {
+            this.messageText = messageText;
+        }
+
+    }
+
+    public static class TwitterUser {
+
+        private String screenName;
+        private String lang = "en";
+        private int friendsCount;
+        private int statusesCount;
+        private String name;
+        private int followersCount;
+
+        public TwitterUser() {
+
+        }
+
+        public TwitterUser(String screenName, int friendsCount, int statusesCount, String name, int followersCount) {
+            this.screenName = screenName;
+            this.friendsCount = friendsCount;
+            this.statusesCount = statusesCount;
+            this.name = name;
+            this.followersCount = followersCount;
+        }
+
+        public void reset(String screenName, int friendsCount, int statusesCount, String name, int followersCount) {
+            this.screenName = screenName;
+            this.friendsCount = friendsCount;
+            this.statusesCount = statusesCount;
+            this.name = name;
+            this.followersCount = followersCount;
+        }
+
+        public String getScreenName() {
+            return screenName;
+        }
+
+        public int getFriendsCount() {
+            return friendsCount;
+        }
+
+        public int getStatusesCount() {
+            return statusesCount;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public int getFollowersCount() {
+            return followersCount;
+        }
+
+        public String toString() {
+            StringBuilder builder = new StringBuilder();
+            builder.append("{");
+            builder.append("\"screen-name\":" + "\"" + screenName + "\"");
+            builder.append(",");
+            builder.append("\"lang\":" + "\"" + lang + "\"");
+            builder.append(",");
+            builder.append("\"friends_count\":" + friendsCount);
+            builder.append(",");
+            builder.append("\"statuses_count\":" + statusesCount);
+            builder.append(",");
+            builder.append("\"name\":" + "\"" + name + "\"");
+            builder.append(",");
+            builder.append("\"followers_count\":" + followersCount);
+            builder.append("}");
+            return builder.toString();
+        }
+
+    }
+
+    public static class DistributionHandler {
+
+        private final ZipfGenerator zipfGen;
+        private final int totalUsers;
+        private final int totalMessages;
+        private Random random = new Random();
+
+        public DistributionHandler(int totalMessages, double skew, int totalNumUsers) {
+            zipfGen = new ZipfGenerator(totalMessages, skew);
+            totalUsers = totalNumUsers;
+            this.totalMessages = totalMessages;
+        }
+
+        public int getFromDistribution(int rank) {
+            double prob = zipfGen.getProbability(rank);
+            int numMessages = (int) (prob * totalMessages);
+            return numMessages;
+        }
+
+        public static void main(String args[]) {
+            int totalMessages = 1000 * 4070;
+            double skew = 0.5;
+            int totalUsers = 4070;
+            DistributionHandler d = new DistributionHandler(totalMessages, skew, totalUsers);
+            int sum = 0;
+            for (int i = totalUsers; i >= 1; i--) {
+                float contrib = d.getFromDistribution(i);
+                sum += contrib;
+                System.out.println(i + ":" + contrib);
+            }
+
+            System.out.println("SUM" + ":" + sum);
+
+        }
+    }
+
+    public static class ZipfGenerator {
+
+        private Random rnd = new Random(System.currentTimeMillis());
+        private int size;
+        private double skew;
+        private double bottom = 0;
+
+        public ZipfGenerator(int size, double skew) {
+            this.size = size;
+            this.skew = skew;
+            for (int i = 1; i < size; i++) {
+                this.bottom += (1 / Math.pow(i, this.skew));
+            }
+        }
+
+        // the next() method returns an rank id. The frequency of returned rank
+        // ids are follows Zipf distribution.
+        public int next() {
+            int rank;
+            double friquency = 0;
+            double dice;
+            rank = rnd.nextInt(size);
+            friquency = (1.0d / Math.pow(rank, this.skew)) / this.bottom;
+            dice = rnd.nextDouble();
+            while (!(dice < friquency)) {
+                rank = rnd.nextInt(size);
+                friquency = (1.0d / Math.pow(rank, this.skew)) / this.bottom;
+                dice = rnd.nextDouble();
+            }
+            return rank;
+        }
+
+        // This method returns a probability that the given rank occurs.
+        public double getProbability(int rank) {
+            return (1.0d / Math.pow(rank, this.skew)) / this.bottom;
+        }
+
+        public static void main(String[] args) throws IOException {
+            int total = (int) (3.7 * 1000 * 1000);
+            int skew = 2;
+            int numUsers = 1000 * 1000;
+            /*
+             * if (args.length != 2) { System.out.println("usage:" +
+             * "./zipf size skew"); System.exit(-1); }
+             */
+            BufferedWriter buf = new BufferedWriter(new FileWriter(new File("/tmp/zip_output")));
+            ZipfGenerator zipf = new ZipfGenerator(total, skew);
+            double sum = 0;
+            for (int i = 1; i <= numUsers; i++) {
+                double prob = zipf.getProbability(i);
+                double contribution = (double) (prob * total);
+                String contrib = i + ":" + contribution;
+                buf.write(contrib);
+                buf.write("\n");
+                System.out.println(contrib);
+                sum += contribution;
+            }
+            System.out.println("sum is :" + sum);
+        }
+    }
+
+    public static class PartitionElement implements ILibraryElement {
+        private final String name;
+        private final String host;
+        private final int fbUserKeyMin;
+        private final int fbUserKeyMax;
+        private final int twUserKeyMin;
+        private final int twUserKeyMax;
+        private final int fbMessageIdMin;
+        private final int fbMessageIdMax;
+        private final int twMessageIdMin;
+        private final int twMessageIdMax;
+
+        public PartitionElement(String partitionName, String host, int fbUserKeyMin, int fbUserKeyMax,
+                int twUserKeyMin, int twUserKeyMax, int fbMessageIdMin, int fbMessageIdMax, int twMessageIdMin,
+                int twMessageIdMax) {
+            this.name = partitionName;
+            this.host = host;
+            this.fbUserKeyMin = fbUserKeyMin;
+            this.fbUserKeyMax = fbUserKeyMax;
+            this.twUserKeyMin = twUserKeyMax;
+            this.twUserKeyMax = twUserKeyMax;
+            this.twMessageIdMin = twMessageIdMin;
+            this.twMessageIdMax = twMessageIdMax;
+            this.fbMessageIdMin = fbMessageIdMin;
+            this.fbMessageIdMax = fbMessageIdMax;
+        }
+
+        public String toString() {
+            StringBuilder builder = new StringBuilder();
+            builder.append(name);
+            builder.append(" ");
+            builder.append(host);
+            builder.append("\n");
+            builder.append("fbUser:key:min");
+            builder.append(fbUserKeyMin);
+
+            builder.append("\n");
+            builder.append("fbUser:key:max");
+            builder.append(fbUserKeyMax);
+
+            builder.append("\n");
+            builder.append("twUser:key:min");
+            builder.append(twUserKeyMin);
+
+            builder.append("\n");
+            builder.append("twUser:key:max");
+            builder.append(twUserKeyMax);
+
+            builder.append("\n");
+            builder.append("fbMessage:key:min");
+            builder.append(fbMessageIdMin);
+
+            builder.append("\n");
+            builder.append("fbMessage:key:max");
+            builder.append(fbMessageIdMax);
+
+            builder.append("\n");
+            builder.append("twMessage:key:min");
+            builder.append(twMessageIdMin);
+
+            builder.append("\n");
+            builder.append("twMessage:key:max");
+            builder.append(twMessageIdMax);
+
+            builder.append("\n");
+            builder.append("twMessage:key:max");
+            builder.append(twUserKeyMin);
+
+            return new String(builder);
+        }
+
+        @Override
+        public String getName() {
+            return "Partition";
+        }
+
+    }
+
+    interface ILibraryElement {
+
+        public enum ElementType {
+            PARTITION
+        }
+
+        public String getName();
+
+    }
+
+    public static class Configuration {
+
+        private final float numMB;
+        private final String unit;
+
+        private final List<SourcePartition> sourcePartitions;
+        private List<TargetPartition> targetPartitions;
+
+        public Configuration(float numMB, String unit, List<SourcePartition> partitions) throws IOException {
+            this.numMB = numMB;
+            this.unit = unit;
+            this.sourcePartitions = partitions;
+
+        }
+
+        public float getNumMB() {
+            return numMB;
+        }
+
+        public String getUnit() {
+            return unit;
+        }
+
+        public List<SourcePartition> getSourcePartitions() {
+            return sourcePartitions;
+        }
+
+        public List<TargetPartition> getTargetPartitions() {
+            return targetPartitions;
+        }
+
+        public void setTargetPartitions(List<TargetPartition> targetPartitions) {
+            this.targetPartitions = targetPartitions;
+        }
+
+    }
+
+    public static class XMLUtil {
+
+        public static void writeToXML(Configuration conf, String filePath) throws IOException,
+                ParserConfigurationException, TransformerException {
+
+            DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
+            DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
+
+            // root elements
+            Document doc = docBuilder.newDocument();
+            Element rootElement = doc.createElement("Partitions");
+            doc.appendChild(rootElement);
+
+            int index = 0;
+            for (TargetPartition partition : conf.getTargetPartitions()) {
+                writePartitionElement(conf.getSourcePartitions().get(index), partition, rootElement, doc);
+            }
+
+            TransformerFactory transformerFactory = TransformerFactory.newInstance();
+            Transformer transformer = transformerFactory.newTransformer();
+
+            transformer.setOutputProperty(OutputKeys.ENCODING, "utf-8");
+            transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "no");
+            transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+            transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
+
+            DOMSource source = new DOMSource(doc);
+            StreamResult result = new StreamResult(new File(filePath));
+
+            transformer.transform(source, result);
+
+        }
+
+        public static void writePartitionInfo(Configuration conf, String filePath) throws IOException {
+            BufferedWriter bw = new BufferedWriter(new FileWriter(filePath));
+            for (SourcePartition sp : conf.getSourcePartitions()) {
+                bw.write(sp.getHost() + ":" + sp.getName() + ":" + sp.getPath());
+                bw.write("\n");
+            }
+            bw.close();
+        }
+
+        public static Document getDocument(String filePath) throws Exception {
+            File inputFile = new File(filePath);
+            DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+            DocumentBuilder db = dbf.newDocumentBuilder();
+            Document doc = db.parse(inputFile);
+            doc.getDocumentElement().normalize();
+            return doc;
+        }
+
+        public static Configuration getConfiguration(String filePath) throws Exception {
+            Configuration conf = getConfiguration(getDocument(filePath));
+            PartitionMetrics metrics = new PartitionMetrics(conf.getNumMB(), conf.getUnit(), conf.getSourcePartitions()
+                    .size());
+            List<TargetPartition> targetPartitions = getTargetPartitions(metrics, conf.getSourcePartitions());
+            conf.setTargetPartitions(targetPartitions);
+            return conf;
+        }
+
+        public static Configuration getConfiguration(Document document) throws IOException {
+            Element rootEle = document.getDocumentElement();
+            NodeList nodeList = rootEle.getChildNodes();
+            float size = Float.parseFloat(getStringValue((Element) nodeList, "size"));
+            String unit = getStringValue((Element) nodeList, "unit");
+            List<SourcePartition> sourcePartitions = getSourcePartitions(document);
+            return new Configuration(size, unit, sourcePartitions);
+        }
+
+        public static List<SourcePartition> getSourcePartitions(Document document) {
+            Element rootEle = document.getDocumentElement();
+            NodeList nodeList = rootEle.getElementsByTagName("partition");
+            List<SourcePartition> sourcePartitions = new ArrayList<SourcePartition>();
+            for (int i = 0; i < nodeList.getLength(); i++) {
+                Node node = nodeList.item(i);
+                sourcePartitions.add(getSourcePartition((Element) node));
+            }
+            return sourcePartitions;
+        }
+
+        public static SourcePartition getSourcePartition(Element functionElement) {
+            String name = getStringValue(functionElement, "name");
+            String host = getStringValue(functionElement, "host");
+            String path = getStringValue(functionElement, "path");
+            SourcePartition sp = new SourcePartition(name, host, path);
+            return sp;
+        }
+
+        public static String getStringValue(Element element, String tagName) {
+            String textValue = null;
+            NodeList nl = element.getElementsByTagName(tagName);
+            if (nl != null && nl.getLength() > 0) {
+                Element el = (Element) nl.item(0);
+                textValue = el.getFirstChild().getNodeValue();
+            }
+            return textValue;
+        }
+
+        public static PartitionConfiguration getPartitionConfiguration(String filePath, String partitionName)
+                throws Exception {
+            PartitionConfiguration pconf = getPartitionConfigurations(getDocument(filePath), partitionName);
+            return pconf;
+        }
+
+        public static PartitionConfiguration getPartitionConfigurations(Document document, String partitionName)
+                throws IOException {
+
+            Element rootEle = document.getDocumentElement();
+            NodeList nodeList = rootEle.getElementsByTagName("Partition");
+
+            for (int i = 0; i < nodeList.getLength(); i++) {
+                Node node = nodeList.item(i);
+                Element nodeElement = (Element) node;
+                String name = getStringValue(nodeElement, "name");
+                if (!name.equalsIgnoreCase(partitionName)) {
+                    continue;
+                }
+                String host = getStringValue(nodeElement, "host");
+                String path = getStringValue(nodeElement, "path");
+
+                String fbUserKeyMin = getStringValue(nodeElement, "fbUserKeyMin");
+                String fbUserKeyMax = getStringValue(nodeElement, "fbUserKeyMax");
+                String twUserKeyMin = getStringValue(nodeElement, "twUserKeyMin");
+                String twUserKeyMax = getStringValue(nodeElement, "twUserKeyMax");
+                String fbMessageKeyMin = getStringValue(nodeElement, "fbMessageKeyMin");
+
+                String fbMessageKeyMax = getStringValue(nodeElement, "fbMessageKeyMax");
+                String twMessageKeyMin = getStringValue(nodeElement, "twMessageKeyMin");
+                String twMessageKeyMax = getStringValue(nodeElement, "twMessageKeyMax");
+                String numCommonUsers = getStringValue(nodeElement, "numCommonUsers");
+
+                SourcePartition sp = new SourcePartition(name, host, path);
+
+                TargetPartition tp = new TargetPartition(partitionName, host, path, Integer.parseInt(fbUserKeyMin),
+                        Integer.parseInt(fbUserKeyMax), Integer.parseInt(twUserKeyMin), Integer.parseInt(twUserKeyMax),
+                        Integer.parseInt(fbMessageKeyMin), Integer.parseInt(fbMessageKeyMax),
+                        Integer.parseInt(twMessageKeyMin), Integer.parseInt(twMessageKeyMax),
+                        Integer.parseInt(numCommonUsers));
+                PartitionConfiguration pc = new PartitionConfiguration(sp, tp);
+                return pc;
+            }
+            return null;
+        }
+
+        public static List<TargetPartition> getTargetPartitions(PartitionMetrics metrics,
+                List<SourcePartition> sourcePartitions) {
+            List<TargetPartition> partitions = new ArrayList<TargetPartition>();
+            int fbUserKeyMin = 1;
+            int twUserKeyMin = 1;
+            int fbMessageIdMin = 1;
+            int twMessageIdMin = 1;
+
+            for (SourcePartition sp : sourcePartitions) {
+                int fbUserKeyMax = fbUserKeyMin + metrics.getFbOnlyUsers() + metrics.getCommonUsers() - 1;
+                int twUserKeyMax = twUserKeyMin + metrics.getTwitterOnlyUsers() + metrics.getCommonUsers() - 1;
+
+                int fbMessageIdMax = fbMessageIdMin + metrics.getFbMessages() - 1;
+                int twMessageIdMax = twMessageIdMin + metrics.getTwMessages() - 1;
+                TargetPartition pe = new TargetPartition(sp.getName(), sp.getHost(), sp.getPath(), fbUserKeyMin,
+                        fbUserKeyMax, twUserKeyMin, twUserKeyMax, fbMessageIdMin, fbMessageIdMax, twMessageIdMin,
+                        twMessageIdMax, metrics.getCommonUsers());
+                partitions.add(pe);
+
+                fbUserKeyMin = fbUserKeyMax + 1;
+                twUserKeyMin = twUserKeyMax + 1;
+
+                fbMessageIdMin = fbMessageIdMax + 1;
+                twMessageIdMin = twMessageIdMax + 1;
+            }
+
+            return partitions;
+        }
+
+        public static void writePartitionElement(SourcePartition sourcePartition, TargetPartition partition,
+                Element rootElement, Document doc) {
+            // staff elements
+            Element pe = doc.createElement("Partition");
+            rootElement.appendChild(pe);
+
+            // name element
+            Element name = doc.createElement("name");
+            name.appendChild(doc.createTextNode("" + partition.getName()));
+            pe.appendChild(name);
+
+            // host element
+            Element host = doc.createElement("host");
+            host.appendChild(doc.createTextNode("" + partition.getHost()));
+            pe.appendChild(host);
+
+            // path element
+            Element path = doc.createElement("path");
+            path.appendChild(doc.createTextNode("" + partition.getPath()));
+            pe.appendChild(path);
+
+            // fbUserKeyMin element
+            Element fbUserKeyMin = doc.createElement("fbUserKeyMin");
+            fbUserKeyMin.appendChild(doc.createTextNode("" + partition.getFbUserKeyMin()));
+            pe.appendChild(fbUserKeyMin);
+
+            // fbUserKeyMax element
+            Element fbUserKeyMax = doc.createElement("fbUserKeyMax");
+            fbUserKeyMax.appendChild(doc.createTextNode("" + partition.getFbUserKeyMax()));
+            pe.appendChild(fbUserKeyMax);
+
+            // twUserKeyMin element
+            Element twUserKeyMin = doc.createElement("twUserKeyMin");
+            twUserKeyMin.appendChild(doc.createTextNode("" + partition.getTwUserKeyMin()));
+            pe.appendChild(twUserKeyMin);
+
+            // twUserKeyMax element
+            Element twUserKeyMax = doc.createElement("twUserKeyMax");
+            twUserKeyMax.appendChild(doc.createTextNode("" + partition.getTwUserKeyMax()));
+            pe.appendChild(twUserKeyMax);
+
+            // fbMessgeKeyMin element
+            Element fbMessageKeyMin = doc.createElement("fbMessageKeyMin");
+            fbMessageKeyMin.appendChild(doc.createTextNode("" + partition.getFbMessageIdMin()));
+            pe.appendChild(fbMessageKeyMin);
+
+            // fbMessgeKeyMin element
+            Element fbMessageKeyMax = doc.createElement("fbMessageKeyMax");
+            fbMessageKeyMax.appendChild(doc.createTextNode("" + partition.getFbMessageIdMax()));
+            pe.appendChild(fbMessageKeyMax);
+
+            // twMessgeKeyMin element
+            Element twMessageKeyMin = doc.createElement("twMessageKeyMin");
+            twMessageKeyMin.appendChild(doc.createTextNode("" + partition.getTwMessageIdMin()));
+            pe.appendChild(twMessageKeyMin);
+
+            // twMessgeKeyMin element
+            Element twMessageKeyMax = doc.createElement("twMessageKeyMax");
+            twMessageKeyMax.appendChild(doc.createTextNode("" + partition.getTwMessageIdMax()));
+            pe.appendChild(twMessageKeyMax);
+
+            // twMessgeKeyMin element
+            Element numCommonUsers = doc.createElement("numCommonUsers");
+            numCommonUsers.appendChild(doc.createTextNode("" + partition.getCommonUsers()));
+            pe.appendChild(numCommonUsers);
+
+        }
+
+        public static void main(String args[]) throws Exception {
+            String confFile = "/Users/rgrove1/work/research/asterix/icde/data-gen/conf/conf.xml";
+            String outputPath = "/Users/rgrove1/work/research/asterix/icde/data-gen/output/conf-output.xml";
+            Configuration conf = getConfiguration(confFile);
+            writeToXML(conf, outputPath);
+        }
+
+    }
+
+    public static class Date {
+
+        private int day;
+        private int month;
+        private int year;
+
+        public Date(int month, int day, int year) {
+            this.month = month;
+            this.day = day;
+            this.year = year;
+        }
+
+        public void reset(int month, int day, int year) {
+            this.month = month;
+            this.day = day;
+            this.year = year;
+        }
+
+        public int getDay() {
+            return day;
+        }
+
+        public int getMonth() {
+            return month;
+        }
+
+        public int getYear() {
+            return year;
+        }
+
+        public Date() {
+        }
+
+        public String toString() {
+            StringBuilder builder = new StringBuilder();
+            builder.append("date");
+            builder.append("(\"");
+            builder.append(year);
+            builder.append("-");
+            builder.append(month < 10 ? "0" + month : "" + month);
+            builder.append("-");
+            builder.append(day < 10 ? "0" + day : "" + day);
+            builder.append("\")");
+            return builder.toString();
+        }
+
+        public void setDay(int day) {
+            this.day = day;
+        }
+
+        public void setMonth(int month) {
+            this.month = month;
+        }
+
+        public void setYear(int year) {
+            this.year = year;
+        }
+    }
+
+    public static class PartitionMetrics {
+
+        private final int fbMessages;
+        private final int twMessages;
+
+        private final int fbOnlyUsers;
+        private final int twitterOnlyUsers;
+        private final int commonUsers;
+
+        public PartitionMetrics(float number, String unit, int numPartitions) throws IOException {
+
+            int factor = 0;
+            if (unit.equalsIgnoreCase("MB")) {
+                factor = 1024 * 1024;
+            } else if (unit.equalsIgnoreCase("GB")) {
+                factor = 1024 * 1024 * 1024;
+            } else if (unit.equalsIgnoreCase("TB")) {
+                factor = 1024 * 1024 * 1024 * 1024;
+            } else
+                throw new IOException("Invalid unit:" + unit);
+
+            fbMessages = (int) (((number * factor * 0.80) / 258) / numPartitions);
+            twMessages = (int) (fbMessages * 1.1 / 0.35);
+
+            fbOnlyUsers = (int) ((number * factor * 0.20 * 0.0043)) / numPartitions;
+            twitterOnlyUsers = (int) (0.25 * fbOnlyUsers);
+            commonUsers = (int) (0.1 * fbOnlyUsers);
+        }
+
+        public int getFbMessages() {
+            return fbMessages;
+        }
+
+        public int getTwMessages() {
+            return twMessages;
+        }
+
+        public int getFbOnlyUsers() {
+            return fbOnlyUsers;
+        }
+
+        public int getTwitterOnlyUsers() {
+            return twitterOnlyUsers;
+        }
+
+        public int getCommonUsers() {
+            return commonUsers;
+        }
+
+    }
+
+    public static String[] lastNames = { "Hoopengarner", "Harrow", "Gardner", "Blyant", "Best", "Buttermore", "Gronko",
+            "Mayers", "Countryman", "Neely", "Ruhl", "Taggart", "Bash", "Cason", "Hil", "Zalack", "Mingle", "Carr",
+            "Rohtin", "Wardle", "Pullman", "Wire", "Kellogg", "Hiles", "Keppel", "Bratton", "Sutton", "Wickes",
+            "Muller", "Friedline", "Llora", "Elizabeth", "Anderson", "Gaskins", "Rifler", "Vinsant", "Stanfield",
+            "Black", "Guest", "Hujsak", "Carter", "Weidemann", "Hays", "Patton", "Hayhurst", "Paynter", "Cressman",
+            "Fiddler", "Evans", "Sherlock", "Woodworth", "Jackson", "Bloise", "Schneider", "Ring", "Kepplinger",
+            "James", "Moon", "Bennett", "Bashline", "Ryals", "Zeal", "Christman", "Milliron", "Nash", "Ewing", "Camp",
+            "Mason", "Richardson", "Bowchiew", "Hahn", "Wilson", "Wood", "Toyley", "Williamson", "Lafortune", "Errett",
+            "Saltser", "Hirleman", "Brindle", "Newbiggin", "Ulery", "Lambert", "Shick", "Kuster", "Moore", "Finck",
+            "Powell", "Jolce", "Townsend", "Sauter", "Cowher", "Wolfe", "Cavalet", "Porter", "Laborde", "Ballou",
+            "Murray", "Stoddard", "Pycroft", "Milne", "King", "Todd", "Staymates", "Hall", "Romanoff", "Keilbach",
+            "Sandford", "Hamilton", "Fye", "Kline", "Weeks", "Mcelroy", "Mccullough", "Bryant", "Hill", "Moore",
+            "Ledgerwood", "Prevatt", "Eckert", "Read", "Hastings", "Doverspike", "Allshouse", "Bryan", "Mccallum",
+            "Lombardi", "Mckendrick", "Cattley", "Barkley", "Steiner", "Finlay", "Priebe", "Armitage", "Hall", "Elder",
+            "Erskine", "Hatcher", "Walker", "Pearsall", "Dunkle", "Haile", "Adams", "Miller", "Newbern", "Basinger",
+            "Fuhrer", "Brinigh", "Mench", "Blackburn", "Bastion", "Mccune", "Bridger", "Hynes", "Quinn", "Courtney",
+            "Geddinge", "Field", "Seelig", "Cable", "Earhart", "Harshman", "Roby", "Beals", "Berry", "Reed", "Hector",
+            "Pittman", "Haverrman", "Kalp", "Briner", "Joghs", "Cowart", "Close", "Wynne", "Harden", "Weldy",
+            "Stephenson", "Hildyard", "Moberly", "Wells", "Mackendoerfer", "Fisher", "Oppie", "Oneal", "Churchill",
+            "Keister", "Alice", "Tavoularis", "Fisher", "Hair", "Burns", "Veith", "Wile", "Fuller", "Fields", "Clark",
+            "Randolph", "Stone", "Mcclymonds", "Holtzer", "Donkin", "Wilkinson", "Rosensteel", "Albright", "Stahl",
+            "Fox", "Kadel", "Houser", "Hanseu", "Henderson", "Davis", "Bicknell", "Swain", "Mercer", "Holdeman",
+            "Enderly", "Caesar", "Margaret", "Munshower", "Elless", "Lucy", "Feufer", "Schofield", "Graham",
+            "Blatenberger", "Benford", "Akers", "Campbell", "Ann", "Sadley", "Ling", "Gongaware", "Schmidt", "Endsley",
+            "Groah", "Flanders", "Reichard", "Lowstetter", "Sandblom", "Griffis", "Basmanoff", "Coveney", "Hawker",
+            "Archibald", "Hutton", "Barnes", "Diegel", "Raybould", "Focell", "Breitenstein", "Murray", "Chauvin",
+            "Busk", "Pheleps", "Teagarden", "Northey", "Baumgartner", "Fleming", "Harris", "Parkinson", "Carpenter",
+            "Whirlow", "Bonner", "Wortman", "Rogers", "Scott", "Lowe", "Mckee", "Huston", "Bullard", "Throckmorton",
+            "Rummel", "Mathews", "Dull", "Saline", "Tue", "Woolery", "Lalty", "Schrader", "Ramsey", "Eisenmann",
+            "Philbrick", "Sybilla", "Wallace", "Fonblanque", "Paul", "Orbell", "Higgens", "Casteel", "Franks",
+            "Demuth", "Eisenman", "Hay", "Robinson", "Fischer", "Hincken", "Wylie", "Leichter", "Bousum",
+            "Littlefield", "Mcdonald", "Greif", "Rhodes", "Wall", "Steele", "Baldwin", "Smith", "Stewart", "Schere",
+            "Mary", "Aultman", "Emrick", "Guess", "Mitchell", "Painter", "Aft", "Hasely", "Weldi", "Loewentsein",
+            "Poorbaugh", "Kepple", "Noton", "Judge", "Jackson", "Style", "Adcock", "Diller", "Marriman", "Johnston",
+            "Children", "Monahan", "Ehret", "Shaw", "Congdon", "Pinney", "Millard", "Crissman", "Tanner", "Rose",
+            "Knisely", "Cypret", "Sommer", "Poehl", "Hardie", "Bender", "Overholt", "Gottwine", "Beach", "Leslie",
+            "Trevithick", "Langston", "Magor", "Shotts", "Howe", "Hunter", "Cross", "Kistler", "Dealtry", "Christner",
+            "Pennington", "Thorley", "Eckhardstein", "Van", "Stroh", "Stough", "Stall", "Beedell", "Shea", "Garland",
+            "Mays", "Pritchard", "Frankenberger", "Rowley", "Lane", "Baum", "Alliman", "Park", "Jardine", "Butler",
+            "Cherry", "Kooser", "Baxter", "Billimek", "Downing", "Hurst", "Wood", "Baird", "Watkins", "Edwards",
+            "Kemerer", "Harding", "Owens", "Eiford", "Keener", "Garneis", "Fiscina", "Mang", "Draudy", "Mills",
+            "Gibson", "Reese", "Todd", "Ramos", "Levett", "Wilks", "Ward", "Mosser", "Dunlap", "Kifer", "Christopher",
+            "Ashbaugh", "Wynter", "Rawls", "Cribbs", "Haynes", "Thigpen", "Schreckengost", "Bishop", "Linton",
+            "Chapman", "James", "Jerome", "Hook", "Omara", "Houston", "Maclagan", "Sandys", "Pickering", "Blois",
+            "Dickson", "Kemble", "Duncan", "Woodward", "Southern", "Henley", "Treeby", "Cram", "Elsas", "Driggers",
+            "Warrick", "Overstreet", "Hindman", "Buck", "Sulyard", "Wentzel", "Swink", "Butt", "Schaeffer",
+            "Hoffhants", "Bould", "Willcox", "Lotherington", "Bagley", "Graff", "White", "Wheeler", "Sloan",
+            "Rodacker", "Hanford", "Jowers", "Kunkle", "Cass", "Powers", "Gilman", "Mcmichaels", "Hobbs", "Herndon",
+            "Prescott", "Smail", "Mcdonald", "Biery", "Orner", "Richards", "Mueller", "Isaman", "Bruxner", "Goodman",
+            "Barth", "Turzanski", "Vorrasi", "Stainforth", "Nehling", "Rahl", "Erschoff", "Greene", "Mckinnon",
+            "Reade", "Smith", "Pery", "Roose", "Greenwood", "Weisgarber", "Curry", "Holts", "Zadovsky", "Parrish",
+            "Putnam", "Munson", "Mcindoe", "Nickolson", "Brooks", "Bollinger", "Stroble", "Siegrist", "Fulton",
+            "Tomey", "Zoucks", "Roberts", "Otis", "Clarke", "Easter", "Johnson", "Fylbrigg", "Taylor", "Swartzbaugh",
+            "Weinstein", "Gadow", "Sayre", "Marcotte", "Wise", "Atweeke", "Mcfall", "Napier", "Eisenhart", "Canham",
+            "Sealis", "Baughman", "Gertraht", "Losey", "Laurence", "Eva", "Pershing", "Kern", "Pirl", "Rega",
+            "Sanborn", "Kanaga", "Sanders", "Anderson", "Dickinson", "Osteen", "Gettemy", "Crom", "Snyder", "Reed",
+            "Laurenzi", "Riggle", "Tillson", "Fowler", "Raub", "Jenner", "Koepple", "Soames", "Goldvogel", "Dimsdale",
+            "Zimmer", "Giesen", "Baker", "Beail", "Mortland", "Bard", "Sanner", "Knopsnider", "Jenkins", "Bailey",
+            "Werner", "Barrett", "Faust", "Agg", "Tomlinson", "Williams", "Little", "Greenawalt", "Wells", "Wilkins",
+            "Gisiko", "Bauerle", "Harrold", "Prechtl", "Polson", "Faast", "Winton", "Garneys", "Peters", "Potter",
+            "Porter", "Tennant", "Eve", "Dugger", "Jones", "Burch", "Cowper", "Whittier" };
+
+    public static String[] firstNames = { "Albert", "Jacquelin", "Dona", "Alia", "Mayme", "Genoveva", "Emma", "Lena",
+            "Melody", "Vilma", "Katelyn", "Jeremy", "Coral", "Leann", "Lita", "Gilda", "Kayla", "Alvina", "Maranda",
+            "Verlie", "Khadijah", "Karey", "Patrice", "Kallie", "Corey", "Mollie", "Daisy", "Melanie", "Sarita",
+            "Nichole", "Pricilla", "Terresa", "Berneice", "Arianne", "Brianne", "Lavinia", "Ulrike", "Lesha", "Adell",
+            "Ardelle", "Marisha", "Laquita", "Karyl", "Maryjane", "Kendall", "Isobel", "Raeann", "Heike", "Barbera",
+            "Norman", "Yasmine", "Nevada", "Mariam", "Edith", "Eugena", "Lovie", "Maren", "Bennie", "Lennie", "Tamera",
+            "Crystal", "Randi", "Anamaria", "Chantal", "Jesenia", "Avis", "Shela", "Randy", "Laurena", "Sharron",
+            "Christiane", "Lorie", "Mario", "Elizabeth", "Reina", "Adria", "Lakisha", "Brittni", "Azzie", "Dori",
+            "Shaneka", "Asuncion", "Katheryn", "Laurice", "Sharita", "Krystal", "Reva", "Inger", "Alpha", "Makeda",
+            "Anabel", "Loni", "Tiara", "Meda", "Latashia", "Leola", "Chin", "Daisey", "Ivory", "Amalia", "Logan",
+            "Tyler", "Kyong", "Carolann", "Maryetta", "Eufemia", "Anya", "Doreatha", "Lorna", "Rutha", "Ehtel",
+            "Debbie", "Chassidy", "Sang", "Christa", "Lottie", "Chun", "Karine", "Peggie", "Amina", "Melany", "Alayna",
+            "Scott", "Romana", "Naomi", "Christiana", "Salena", "Taunya", "Mitsue", "Regina", "Chelsie", "Charity",
+            "Dacia", "Aletha", "Latosha", "Lia", "Tamica", "Chery", "Bianca", "Shu", "Georgianne", "Myriam", "Austin",
+            "Wan", "Mallory", "Jana", "Georgie", "Jenell", "Kori", "Vicki", "Delfina", "June", "Mellisa", "Catherina",
+            "Claudie", "Tynisha", "Dayle", "Enriqueta", "Belen", "Pia", "Sarai", "Rosy", "Renay", "Kacie", "Frieda",
+            "Cayla", "Elissa", "Claribel", "Sabina", "Mackenzie", "Raina", "Cira", "Mitzie", "Aubrey", "Serafina",
+            "Maria", "Katharine", "Esperanza", "Sung", "Daria", "Billye", "Stefanie", "Kasha", "Holly", "Suzanne",
+            "Inga", "Flora", "Andria", "Genevie", "Eladia", "Janet", "Erline", "Renna", "Georgeanna", "Delorse",
+            "Elnora", "Rudy", "Rima", "Leanora", "Letisha", "Love", "Alverta", "Pinkie", "Domonique", "Jeannie",
+            "Jose", "Jacqueline", "Tara", "Lily", "Erna", "Tennille", "Galina", "Tamala", "Kirby", "Nichelle",
+            "Myesha", "Farah", "Santa", "Ludie", "Kenia", "Yee", "Micheline", "Maryann", "Elaina", "Ethelyn",
+            "Emmaline", "Shanell", "Marina", "Nila", "Alane", "Shakira", "Dorris", "Belinda", "Elois", "Barbie",
+            "Carita", "Gisela", "Lura", "Fransisca", "Helga", "Peg", "Leonarda", "Earlie", "Deetta", "Jacquetta",
+            "Blossom", "Kayleigh", "Deloras", "Keshia", "Christinia", "Dulce", "Bernie", "Sheba", "Lashanda", "Tula",
+            "Claretta", "Kary", "Jeanette", "Lupita", "Lenora", "Hisako", "Sherise", "Glynda", "Adela", "Chia",
+            "Sudie", "Mindy", "Caroyln", "Lindsey", "Xiomara", "Mercedes", "Onie", "Loan", "Alexis", "Tommie",
+            "Donette", "Monica", "Soo", "Camellia", "Lavera", "Valery", "Ariana", "Sophia", "Loris", "Ginette",
+            "Marielle", "Tari", "Julissa", "Alesia", "Suzanna", "Emelda", "Erin", "Ladawn", "Sherilyn", "Candice",
+            "Nereida", "Fairy", "Carl", "Joel", "Marilee", "Gracia", "Cordie", "So", "Shanita", "Drew", "Cassie",
+            "Sherie", "Marget", "Norma", "Delois", "Debera", "Chanelle", "Catarina", "Aracely", "Carlene", "Tricia",
+            "Aleen", "Katharina", "Marguerita", "Guadalupe", "Margorie", "Mandie", "Kathe", "Chong", "Sage", "Faith",
+            "Maryrose", "Stephany", "Ivy", "Pauline", "Susie", "Cristen", "Jenifer", "Annette", "Debi", "Karmen",
+            "Luci", "Shayla", "Hope", "Ocie", "Sharie", "Tami", "Breana", "Kerry", "Rubye", "Lashay", "Sondra",
+            "Katrice", "Brunilda", "Cortney", "Yan", "Zenobia", "Penni", "Addie", "Lavona", "Noel", "Anika",
+            "Herlinda", "Valencia", "Bunny", "Tory", "Victoria", "Carrie", "Mikaela", "Wilhelmina", "Chung",
+            "Hortencia", "Gerda", "Wen", "Ilana", "Sibyl", "Candida", "Victorina", "Chantell", "Casie", "Emeline",
+            "Dominica", "Cecila", "Delora", "Miesha", "Nova", "Sally", "Ronald", "Charlette", "Francisca", "Mina",
+            "Jenna", "Loraine", "Felisa", "Lulu", "Page", "Lyda", "Babara", "Flor", "Walter", "Chan", "Sherika",
+            "Kala", "Luna", "Vada", "Syreeta", "Slyvia", "Karin", "Renata", "Robbi", "Glenda", "Delsie", "Lizzie",
+            "Genia", "Caitlin", "Bebe", "Cory", "Sam", "Leslee", "Elva", "Caren", "Kasie", "Leticia", "Shannan",
+            "Vickey", "Sandie", "Kyle", "Chang", "Terrilyn", "Sandra", "Elida", "Marketta", "Elsy", "Tu", "Carman",
+            "Ashlie", "Vernia", "Albertine", "Vivian", "Elba", "Bong", "Margy", "Janetta", "Xiao", "Teofila", "Danyel",
+            "Nickole", "Aleisha", "Tera", "Cleotilde", "Dara", "Paulita", "Isela", "Maricela", "Rozella", "Marivel",
+            "Aurora", "Melissa", "Carylon", "Delinda", "Marvella", "Candelaria", "Deidre", "Tawanna", "Myrtie",
+            "Milagro", "Emilie", "Coretta", "Ivette", "Suzann", "Ammie", "Lucina", "Lory", "Tena", "Eleanor",
+            "Cherlyn", "Tiana", "Brianna", "Myra", "Flo", "Carisa", "Kandi", "Erlinda", "Jacqulyn", "Fermina", "Riva",
+            "Palmira", "Lindsay", "Annmarie", "Tamiko", "Carline", "Amelia", "Quiana", "Lashawna", "Veola", "Belva",
+            "Marsha", "Verlene", "Alex", "Leisha", "Camila", "Mirtha", "Melva", "Lina", "Arla", "Cythia", "Towanda",
+            "Aracelis", "Tasia", "Aurore", "Trinity", "Bernadine", "Farrah", "Deneen", "Ines", "Betty", "Lorretta",
+            "Dorethea", "Hertha", "Rochelle", "Juli", "Shenika", "Yung", "Lavon", "Deeanna", "Nakia", "Lynnette",
+            "Dinorah", "Nery", "Elene", "Carolee", "Mira", "Franchesca", "Lavonda", "Leida", "Paulette", "Dorine",
+            "Allegra", "Keva", "Jeffrey", "Bernardina", "Maryln", "Yoko", "Faviola", "Jayne", "Lucilla", "Charita",
+            "Ewa", "Ella", "Maggie", "Ivey", "Bettie", "Jerri", "Marni", "Bibi", "Sabrina", "Sarah", "Marleen",
+            "Katherin", "Remona", "Jamika", "Antonina", "Oliva", "Lajuana", "Fonda", "Sigrid", "Yael", "Billi",
+            "Verona", "Arminda", "Mirna", "Tesha", "Katheleen", "Bonita", "Kamilah", "Patrica", "Julio", "Shaina",
+            "Mellie", "Denyse", "Deandrea", "Alena", "Meg", "Kizzie", "Krissy", "Karly", "Alleen", "Yahaira", "Lucie",
+            "Karena", "Elaine", "Eloise", "Buena", "Marianela", "Renee", "Nan", "Carolynn", "Windy", "Avril", "Jane",
+            "Vida", "Thea", "Marvel", "Rosaline", "Tifany", "Robena", "Azucena", "Carlota", "Mindi", "Andera", "Jenny",
+            "Courtney", "Lyndsey", "Willette", "Kristie", "Shaniqua", "Tabatha", "Ngoc", "Una", "Marlena", "Louetta",
+            "Vernie", "Brandy", "Jacquelyne", "Jenelle", "Elna", "Erminia", "Ida", "Audie", "Louis", "Marisol",
+            "Shawana", "Harriette", "Karol", "Kitty", "Esmeralda", "Vivienne", "Eloisa", "Iris", "Jeanice", "Cammie",
+            "Jacinda", "Shena", "Floy", "Theda", "Lourdes", "Jayna", "Marg", "Kati", "Tanna", "Rosalyn", "Maxima",
+            "Soon", "Angelika", "Shonna", "Merle", "Kassandra", "Deedee", "Heidi", "Marti", "Renae", "Arleen",
+            "Alfredia", "Jewell", "Carley", "Pennie", "Corina", "Tonisha", "Natividad", "Lilliana", "Darcie", "Shawna",
+            "Angel", "Piedad", "Josefa", "Rebbeca", "Natacha", "Nenita", "Petrina", "Carmon", "Chasidy", "Temika",
+            "Dennise", "Renetta", "Augusta", "Shirlee", "Valeri", "Casimira", "Janay", "Berniece", "Deborah", "Yaeko",
+            "Mimi", "Digna", "Irish", "Cher", "Yong", "Lucila", "Jimmie", "Junko", "Lezlie", "Waneta", "Sandee",
+            "Marquita", "Eura", "Freeda", "Annabell", "Laree", "Jaye", "Wendy", "Toshia", "Kylee", "Aleta", "Emiko",
+            "Clorinda", "Sixta", "Audrea", "Juanita", "Birdie", "Reita", "Latanya", "Nia", "Leora", "Laurine",
+            "Krysten", "Jerrie", "Chantel", "Ira", "Sena", "Andre", "Jann", "Marla", "Precious", "Katy", "Gabrielle",
+            "Yvette", "Brook", "Shirlene", "Eldora", "Laura", "Milda", "Euna", "Jettie", "Debora", "Lise", "Edythe",
+            "Leandra", "Shandi", "Araceli", "Johanne", "Nieves", "Denese", "Carmelita", "Nohemi", "Annice", "Natalie",
+            "Yolande", "Jeffie", "Vashti", "Vickie", "Obdulia", "Youlanda", "Lupe", "Tomoko", "Monserrate", "Domitila",
+            "Etsuko", "Adrienne", "Lakesha", "Melissia", "Odessa", "Meagan", "Veronika", "Jolyn", "Isabelle", "Leah",
+            "Rhiannon", "Gianna", "Audra", "Sommer", "Renate", "Perla", "Thao", "Myong", "Lavette", "Mark", "Emilia",
+            "Ariane", "Karl", "Dorie", "Jacquie", "Mia", "Malka", "Shenita", "Tashina", "Christine", "Cherri", "Roni",
+            "Fran", "Mildred", "Sara", "Clarissa", "Fredia", "Elease", "Samuel", "Earlene", "Vernita", "Mae", "Concha",
+            "Renea", "Tamekia", "Hye", "Ingeborg", "Tessa", "Kelly", "Kristin", "Tam", "Sacha", "Kanisha", "Jillian",
+            "Tiffanie", "Ashlee", "Madelyn", "Donya", "Clementine", "Mickie", "My", "Zena", "Terrie", "Samatha",
+            "Gertie", "Tarra", "Natalia", "Sharlene", "Evie", "Shalon", "Rosalee", "Numbers", "Jodi", "Hattie",
+            "Naoma", "Valene", "Whitley", "Claude", "Alline", "Jeanne", "Camie", "Maragret", "Viola", "Kris", "Marlo",
+            "Arcelia", "Shari", "Jalisa", "Corrie", "Eleonor", "Angelyn", "Merry", "Lauren", "Melita", "Gita",
+            "Elenor", "Aurelia", "Janae", "Lyndia", "Margeret", "Shawanda", "Rolande", "Shirl", "Madeleine", "Celinda",
+            "Jaleesa", "Shemika", "Joye", "Tisa", "Trudie", "Kathrine", "Clarita", "Dinah", "Georgia", "Antoinette",
+            "Janis", "Suzette", "Sherri", "Herta", "Arie", "Hedy", "Cassi", "Audrie", "Caryl", "Jazmine", "Jessica",
+            "Beverly", "Elizbeth", "Marylee", "Londa", "Fredericka", "Argelia", "Nana", "Donnette", "Damaris",
+            "Hailey", "Jamee", "Kathlene", "Glayds", "Lydia", "Apryl", "Verla", "Adam", "Concepcion", "Zelda",
+            "Shonta", "Vernice", "Detra", "Meghann", "Sherley", "Sheri", "Kiyoko", "Margarita", "Adaline", "Mariela",
+            "Velda", "Ailene", "Juliane", "Aiko", "Edyth", "Cecelia", "Shavon", "Florance", "Madeline", "Rheba",
+            "Deann", "Ignacia", "Odelia", "Heide", "Mica", "Jennette", "Maricruz", "Ouida", "Darcy", "Laure",
+            "Justina", "Amada", "Laine", "Cruz", "Sunny", "Francene", "Roxanna", "Nam", "Nancie", "Deanna", "Letty",
+            "Britni", "Kazuko", "Lacresha", "Simon", "Caleb", "Milton", "Colton", "Travis", "Miles", "Jonathan",
+            "Logan", "Rolf", "Emilio", "Roberto", "Marcus", "Tim", "Delmar", "Devon", "Kurt", "Edward", "Jeffrey",
+            "Elvis", "Alfonso", "Blair", "Wm", "Sheldon", "Leonel", "Michal", "Federico", "Jacques", "Leslie",
+            "Augustine", "Hugh", "Brant", "Hong", "Sal", "Modesto", "Curtis", "Jefferey", "Adam", "John", "Glenn",
+            "Vance", "Alejandro", "Refugio", "Lucio", "Demarcus", "Chang", "Huey", "Neville", "Preston", "Bert",
+            "Abram", "Foster", "Jamison", "Kirby", "Erich", "Manual", "Dustin", "Derrick", "Donnie", "Jospeh", "Chris",
+            "Josue", "Stevie", "Russ", "Stanley", "Nicolas", "Samuel", "Waldo", "Jake", "Max", "Ernest", "Reinaldo",
+            "Rene", "Gale", "Morris", "Nathan", "Maximo", "Courtney", "Theodore", "Octavio", "Otha", "Delmer",
+            "Graham", "Dean", "Lowell", "Myles", "Colby", "Boyd", "Adolph", "Jarrod", "Nick", "Mark", "Clinton", "Kim",
+            "Sonny", "Dalton", "Tyler", "Jody", "Orville", "Luther", "Rubin", "Hollis", "Rashad", "Barton", "Vicente",
+            "Ted", "Rick", "Carmine", "Clifton", "Gayle", "Christopher", "Jessie", "Bradley", "Clay", "Theo", "Josh",
+            "Mitchell", "Boyce", "Chung", "Eugenio", "August", "Norbert", "Sammie", "Jerry", "Adan", "Edmundo",
+            "Homer", "Hilton", "Tod", "Kirk", "Emmett", "Milan", "Quincy", "Jewell", "Herb", "Steve", "Carmen",
+            "Bobby", "Odis", "Daron", "Jeremy", "Carl", "Hunter", "Tuan", "Thurman", "Asa", "Brenton", "Shane",
+            "Donny", "Andreas", "Teddy", "Dario", "Cyril", "Hoyt", "Teodoro", "Vincenzo", "Hilario", "Daren",
+            "Agustin", "Marquis", "Ezekiel", "Brendan", "Johnson", "Alden", "Richie", "Granville", "Chad", "Joseph",
+            "Lamont", "Jordon", "Gilberto", "Chong", "Rosendo", "Eddy", "Rob", "Dewitt", "Andre", "Titus", "Russell",
+            "Rigoberto", "Dick", "Garland", "Gabriel", "Hank", "Darius", "Ignacio", "Lazaro", "Johnie", "Mauro",
+            "Edmund", "Trent", "Harris", "Osvaldo", "Marvin", "Judson", "Rodney", "Randall", "Renato", "Richard",
+            "Denny", "Jon", "Doyle", "Cristopher", "Wilson", "Christian", "Jamie", "Roland", "Ken", "Tad", "Romeo",
+            "Seth", "Quinton", "Byron", "Ruben", "Darrel", "Deandre", "Broderick", "Harold", "Ty", "Monroe", "Landon",
+            "Mohammed", "Angel", "Arlen", "Elias", "Andres", "Carlton", "Numbers", "Tony", "Thaddeus", "Issac",
+            "Elmer", "Antoine", "Ned", "Fermin", "Grover", "Benito", "Abdul", "Cortez", "Eric", "Maxwell", "Coy",
+            "Gavin", "Rich", "Andy", "Del", "Giovanni", "Major", "Efren", "Horacio", "Joaquin", "Charles", "Noah",
+            "Deon", "Pasquale", "Reed", "Fausto", "Jermaine", "Irvin", "Ray", "Tobias", "Carter", "Yong", "Jorge",
+            "Brent", "Daniel", "Zane", "Walker", "Thad", "Shaun", "Jaime", "Mckinley", "Bradford", "Nathanial",
+            "Jerald", "Aubrey", "Virgil", "Abel", "Philip", "Chester", "Chadwick", "Dominick", "Britt", "Emmitt",
+            "Ferdinand", "Julian", "Reid", "Santos", "Dwain", "Morgan", "James", "Marion", "Micheal", "Eddie", "Brett",
+            "Stacy", "Kerry", "Dale", "Nicholas", "Darrick", "Freeman", "Scott", "Newton", "Sherman", "Felton",
+            "Cedrick", "Winfred", "Brad", "Fredric", "Dewayne", "Virgilio", "Reggie", "Edgar", "Heriberto", "Shad",
+            "Timmy", "Javier", "Nestor", "Royal", "Lynn", "Irwin", "Ismael", "Jonas", "Wiley", "Austin", "Kieth",
+            "Gonzalo", "Paris", "Earnest", "Arron", "Jarred", "Todd", "Erik", "Maria", "Chauncey", "Neil", "Conrad",
+            "Maurice", "Roosevelt", "Jacob", "Sydney", "Lee", "Basil", "Louis", "Rodolfo", "Rodger", "Roman", "Corey",
+            "Ambrose", "Cristobal", "Sylvester", "Benton", "Franklin", "Marcelo", "Guillermo", "Toby", "Jeramy",
+            "Donn", "Danny", "Dwight", "Clifford", "Valentine", "Matt", "Jules", "Kareem", "Ronny", "Lonny", "Son",
+            "Leopoldo", "Dannie", "Gregg", "Dillon", "Orlando", "Weston", "Kermit", "Damian", "Abraham", "Walton",
+            "Adrian", "Rudolf", "Will", "Les", "Norberto", "Fred", "Tyrone", "Ariel", "Terry", "Emmanuel", "Anderson",
+            "Elton", "Otis", "Derek", "Frankie", "Gino", "Lavern", "Jarod", "Kenny", "Dane", "Keenan", "Bryant",
+            "Eusebio", "Dorian", "Ali", "Lucas", "Wilford", "Jeremiah", "Warner", "Woodrow", "Galen", "Bob",
+            "Johnathon", "Amado", "Michel", "Harry", "Zachery", "Taylor", "Booker", "Hershel", "Mohammad", "Darrell",
+            "Kyle", "Stuart", "Marlin", "Hyman", "Jeffery", "Sidney", "Merrill", "Roy", "Garrett", "Porter", "Kenton",
+            "Giuseppe", "Terrance", "Trey", "Felix", "Buster", "Von", "Jackie", "Linwood", "Darron", "Francisco",
+            "Bernie", "Diego", "Brendon", "Cody", "Marco", "Ahmed", "Antonio", "Vince", "Brooks", "Kendrick", "Ross",
+            "Mohamed", "Jim", "Benny", "Gerald", "Pablo", "Charlie", "Antony", "Werner", "Hipolito", "Minh", "Mel",
+            "Derick", "Armand", "Fidel", "Lewis", "Donnell", "Desmond", "Vaughn", "Guadalupe", "Keneth", "Rodrick",
+            "Spencer", "Chas", "Gus", "Harlan", "Wes", "Carmelo", "Jefferson", "Gerard", "Jarvis", "Haywood", "Hayden",
+            "Sergio", "Gene", "Edgardo", "Colin", "Horace", "Dominic", "Aldo", "Adolfo", "Juan", "Man", "Lenard",
+            "Clement", "Everett", "Hal", "Bryon", "Mason", "Emerson", "Earle", "Laurence", "Columbus", "Lamar",
+            "Douglas", "Ian", "Fredrick", "Marc", "Loren", "Wallace", "Randell", "Noble", "Ricardo", "Rory", "Lindsey",
+            "Boris", "Bill", "Carlos", "Domingo", "Grant", "Craig", "Ezra", "Matthew", "Van", "Rudy", "Danial",
+            "Brock", "Maynard", "Vincent", "Cole", "Damion", "Ellsworth", "Marcel", "Markus", "Rueben", "Tanner",
+            "Reyes", "Hung", "Kennith", "Lindsay", "Howard", "Ralph", "Jed", "Monte", "Garfield", "Avery", "Bernardo",
+            "Malcolm", "Sterling", "Ezequiel", "Kristofer", "Luciano", "Casey", "Rosario", "Ellis", "Quintin",
+            "Trevor", "Miquel", "Jordan", "Arthur", "Carson", "Tyron", "Grady", "Walter", "Jonathon", "Ricky",
+            "Bennie", "Terrence", "Dion", "Dusty", "Roderick", "Isaac", "Rodrigo", "Harrison", "Zack", "Dee", "Devin",
+            "Rey", "Ulysses", "Clint", "Greg", "Dino", "Frances", "Wade", "Franklyn", "Jude", "Bradly", "Salvador",
+            "Rocky", "Weldon", "Lloyd", "Milford", "Clarence", "Alec", "Allan", "Bobbie", "Oswaldo", "Wilfred",
+            "Raleigh", "Shelby", "Willy", "Alphonso", "Arnoldo", "Robbie", "Truman", "Nicky", "Quinn", "Damien",
+            "Lacy", "Marcos", "Parker", "Burt", "Carroll", "Denver", "Buck", "Dong", "Normand", "Billie", "Edwin",
+            "Troy", "Arden", "Rusty", "Tommy", "Kenneth", "Leo", "Claud", "Joel", "Kendall", "Dante", "Milo", "Cruz",
+            "Lucien", "Ramon", "Jarrett", "Scottie", "Deshawn", "Ronnie", "Pete", "Alonzo", "Whitney", "Stefan",
+            "Sebastian", "Edmond", "Enrique", "Branden", "Leonard", "Loyd", "Olin", "Ron", "Rhett", "Frederic",
+            "Orval", "Tyrell", "Gail", "Eli", "Antonia", "Malcom", "Sandy", "Stacey", "Nickolas", "Hosea", "Santo",
+            "Oscar", "Fletcher", "Dave", "Patrick", "Dewey", "Bo", "Vito", "Blaine", "Randy", "Robin", "Winston",
+            "Sammy", "Edwardo", "Manuel", "Valentin", "Stanford", "Filiberto", "Buddy", "Zachariah", "Johnnie",
+            "Elbert", "Paul", "Isreal", "Jerrold", "Leif", "Owen", "Sung", "Junior", "Raphael", "Josef", "Donte",
+            "Allen", "Florencio", "Raymond", "Lauren", "Collin", "Eliseo", "Bruno", "Martin", "Lyndon", "Kurtis",
+            "Salvatore", "Erwin", "Michael", "Sean", "Davis", "Alberto", "King", "Rolland", "Joe", "Tory", "Chase",
+            "Dallas", "Vernon", "Beau", "Terrell", "Reynaldo", "Monty", "Jame", "Dirk", "Florentino", "Reuben", "Saul",
+            "Emory", "Esteban", "Michale", "Claudio", "Jacinto", "Kelley", "Levi", "Andrea", "Lanny", "Wendell",
+            "Elwood", "Joan", "Felipe", "Palmer", "Elmo", "Lawrence", "Hubert", "Rudolph", "Duane", "Cordell",
+            "Everette", "Mack", "Alan", "Efrain", "Trenton", "Bryan", "Tom", "Wilmer", "Clyde", "Chance", "Lou",
+            "Brain", "Justin", "Phil", "Jerrod", "George", "Kris", "Cyrus", "Emery", "Rickey", "Lincoln", "Renaldo",
+            "Mathew", "Luke", "Dwayne", "Alexis", "Jackson", "Gil", "Marty", "Burton", "Emil", "Glen", "Willian",
+            "Clemente", "Keven", "Barney", "Odell", "Reginald", "Aurelio", "Damon", "Ward", "Gustavo", "Harley",
+            "Peter", "Anibal", "Arlie", "Nigel", "Oren", "Zachary", "Scot", "Bud", "Wilbert", "Bart", "Josiah",
+            "Marlon", "Eldon", "Darryl", "Roger", "Anthony", "Omer", "Francis", "Patricia", "Moises", "Chuck",
+            "Waylon", "Hector", "Jamaal", "Cesar", "Julius", "Rex", "Norris", "Ollie", "Isaias", "Quentin", "Graig",
+            "Lyle", "Jeffry", "Karl", "Lester", "Danilo", "Mike", "Dylan", "Carlo", "Ryan", "Leon", "Percy", "Lucius",
+            "Jamel", "Lesley", "Joey", "Cornelius", "Rico", "Arnulfo", "Chet", "Margarito", "Ernie", "Nathanael",
+            "Amos", "Cleveland", "Luigi", "Alfonzo", "Phillip", "Clair", "Elroy", "Alva", "Hans", "Shon", "Gary",
+            "Jesus", "Cary", "Silas", "Keith", "Israel", "Willard", "Randolph", "Dan", "Adalberto", "Claude",
+            "Delbert", "Garry", "Mary", "Larry", "Riley", "Robt", "Darwin", "Barrett", "Steven", "Kelly", "Herschel",
+            "Darnell", "Scotty", "Armando", "Miguel", "Lawerence", "Wesley", "Garth", "Carol", "Micah", "Alvin",
+            "Billy", "Earl", "Pat", "Brady", "Cory", "Carey", "Bernard", "Jayson", "Nathaniel", "Gaylord", "Archie",
+            "Dorsey", "Erasmo", "Angelo", "Elisha", "Long", "Augustus", "Hobert", "Drew", "Stan", "Sherwood",
+            "Lorenzo", "Forrest", "Shawn", "Leigh", "Hiram", "Leonardo", "Gerry", "Myron", "Hugo", "Alvaro", "Leland",
+            "Genaro", "Jamey", "Stewart", "Elden", "Irving", "Olen", "Antone", "Freddy", "Lupe", "Joshua", "Gregory",
+            "Andrew", "Sang", "Wilbur", "Gerardo", "Merlin", "Williams", "Johnny", "Alex", "Tommie", "Jimmy",
+            "Donovan", "Dexter", "Gaston", "Tracy", "Jeff", "Stephen", "Berry", "Anton", "Darell", "Fritz", "Willis",
+            "Noel", "Mariano", "Crawford", "Zoey", "Alex", "Brianna", "Carlie", "Lloyd", "Cal", "Astor", "Randolf",
+            "Magdalene", "Trevelyan", "Terance", "Roy", "Kermit", "Harriett", "Crystal", "Laurinda", "Kiersten",
+            "Phyllida", "Liz", "Bettie", "Rena", "Colten", "Berenice", "Sindy", "Wilma", "Amos", "Candi", "Ritchie",
+            "Dirk", "Kathlyn", "Callista", "Anona", "Flossie", "Sterling", "Calista", "Regan", "Erica", "Jeana",
+            "Keaton", "York", "Nolan", "Daniel", "Benton", "Tommie", "Serenity", "Deanna", "Chas", "Heron", "Marlyn",
+            "Xylia", "Tristin", "Lyndon", "Andriana", "Madelaine", "Maddison", "Leila", "Chantelle", "Audrey",
+            "Connor", "Daley", "Tracee", "Tilda", "Eliot", "Merle", "Linwood", "Kathryn", "Silas", "Alvina",
+            "Phinehas", "Janis", "Alvena", "Zubin", "Gwendolen", "Caitlyn", "Bertram", "Hailee", "Idelle", "Homer",
+            "Jannah", "Delbert", "Rhianna", "Cy", "Jefferson", "Wayland", "Nona", "Tempest", "Reed", "Jenifer",
+            "Ellery", "Nicolina", "Aldous", "Prince", "Lexia", "Vinnie", "Doug", "Alberic", "Kayleen", "Woody",
+            "Rosanne", "Ysabel", "Skyler", "Twyla", "Geordie", "Leta", "Clive", "Aaron", "Scottie", "Celeste", "Chuck",
+            "Erle", "Lallie", "Jaycob", "Ray", "Carrie", "Laurita", "Noreen", "Meaghan", "Ulysses", "Andy", "Drogo",
+            "Dina", "Yasmin", "Mya", "Luvenia", "Urban", "Jacob", "Laetitia", "Sherry", "Love", "Michaela", "Deonne",
+            "Summer", "Brendon", "Sheena", "Mason", "Jayson", "Linden", "Salal", "Darrell", "Diana", "Hudson",
+            "Lennon", "Isador", "Charley", "April", "Ralph", "James", "Mina", "Jolyon", "Laurine", "Monna", "Carita",
+            "Munro", "Elsdon", "Everette", "Radclyffe", "Darrin", "Herbert", "Gawain", "Sheree", "Trudy", "Emmaline",
+            "Kassandra", "Rebecca", "Basil", "Jen", "Don", "Osborne", "Lilith", "Hannah", "Fox", "Rupert", "Paulene",
+            "Darius", "Wally", "Baptist", "Sapphire", "Tia", "Sondra", "Kylee", "Ashton", "Jepson", "Joetta", "Val",
+            "Adela", "Zacharias", "Zola", "Marmaduke", "Shannah", "Posie", "Oralie", "Brittany", "Ernesta", "Raymund",
+            "Denzil", "Daren", "Roosevelt", "Nelson", "Fortune", "Mariel", "Nick", "Jaden", "Upton", "Oz", "Margaux",
+            "Precious", "Albert", "Bridger", "Jimmy", "Nicola", "Rosalynne", "Keith", "Walt", "Della", "Joanna",
+            "Xenia", "Esmeralda", "Major", "Simon", "Rexana", "Stacy", "Calanthe", "Sherley", "Kaitlyn", "Graham",
+            "Ramsey", "Abbey", "Madlyn", "Kelvin", "Bill", "Rue", "Monica", "Caileigh", "Laraine", "Booker", "Jayna",
+            "Greta", "Jervis", "Sherman", "Kendrick", "Tommy", "Iris", "Geffrey", "Kaelea", "Kerr", "Garrick", "Jep",
+            "Audley", "Nic", "Bronte", "Beulah", "Patricia", "Jewell", "Deidra", "Cory", "Everett", "Harper",
+            "Charity", "Godfrey", "Jaime", "Sinclair", "Talbot", "Dayna", "Cooper", "Rosaline", "Jennie", "Eileen",
+            "Latanya", "Corinna", "Roxie", "Caesar", "Charles", "Pollie", "Lindsey", "Sorrel", "Dwight", "Jocelyn",
+            "Weston", "Shyla", "Valorie", "Bessie", "Josh", "Lessie", "Dayton", "Kathi", "Chasity", "Wilton", "Adam",
+            "William", "Ash", "Angela", "Ivor", "Ria", "Jazmine", "Hailey", "Jo", "Silvestra", "Ernie", "Clifford",
+            "Levi", "Matilda", "Quincey", "Camilla", "Delicia", "Phemie", "Laurena", "Bambi", "Lourdes", "Royston",
+            "Chastity", "Lynwood", "Elle", "Brenda", "Phoebe", "Timothy", "Raschelle", "Lilly", "Burt", "Rina",
+            "Rodney", "Maris", "Jaron", "Wilf", "Harlan", "Audra", "Vincent", "Elwyn", "Drew", "Wynter", "Ora",
+            "Lissa", "Virgil", "Xavier", "Chad", "Ollie", "Leyton", "Karolyn", "Skye", "Roni", "Gladys", "Dinah",
+            "Penny", "August", "Osmund", "Whitaker", "Brande", "Cornell", "Phil", "Zara", "Kilie", "Gavin", "Coty",
+            "Randy", "Teri", "Keira", "Pru", "Clemency", "Kelcey", "Nevil", "Poppy", "Gareth", "Christabel", "Bastian",
+            "Wynonna", "Roselyn", "Goddard", "Collin", "Trace", "Neal", "Effie", "Denys", "Virginia", "Richard",
+            "Isiah", "Harrietta", "Gaylord", "Diamond", "Trudi", "Elaine", "Jemmy", "Gage", "Annabel", "Quincy", "Syd",
+            "Marianna", "Philomena", "Aubree", "Kathie", "Jacki", "Kelley", "Bess", "Cecil", "Maryvonne", "Kassidy",
+            "Anselm", "Dona", "Darby", "Jamison", "Daryl", "Darell", "Teal", "Lennie", "Bartholomew", "Katie",
+            "Maybelline", "Kimball", "Elvis", "Les", "Flick", "Harley", "Beth", "Bidelia", "Montague", "Helen", "Ozzy",
+            "Stef", "Debra", "Maxene", "Stefanie", "Russ", "Avril", "Johnathan", "Orson", "Chelsey", "Josephine",
+            "Deshaun", "Wendell", "Lula", "Ferdinanda", "Greg", "Brad", "Kynaston", "Dena", "Russel", "Robertina",
+            "Misti", "Leon", "Anjelica", "Bryana", "Myles", "Judi", "Curtis", "Davin", "Kristia", "Chrysanta",
+            "Hayleigh", "Hector", "Osbert", "Eustace", "Cary", "Tansy", "Cayley", "Maryann", "Alissa", "Ike",
+            "Tranter", "Reina", "Alwilda", "Sidony", "Columbine", "Astra", "Jillie", "Stephania", "Jonah", "Kennedy",
+            "Ferdinand", "Allegria", "Donella", "Kelleigh", "Darian", "Eldreda", "Jayden", "Herbie", "Jake", "Winston",
+            "Vi", "Annie", "Cherice", "Hugo", "Tricia", "Haydee", "Cassarah", "Darden", "Mallory", "Alton", "Hadley",
+            "Romayne", "Lacey", "Ern", "Alayna", "Cecilia", "Seward", "Tilly", "Edgar", "Concordia", "Ibbie", "Dahlia",
+            "Oswin", "Stu", "Brett", "Maralyn", "Kristeen", "Dotty", "Robyn", "Nessa", "Tresha", "Guinevere",
+            "Emerson", "Haze", "Lyn", "Henderson", "Lexa", "Jaylen", "Gail", "Lizette", "Tiara", "Robbie", "Destiny",
+            "Alice", "Livia", "Rosy", "Leah", "Jan", "Zach", "Vita", "Gia", "Micheal", "Rowina", "Alysha", "Bobbi",
+            "Delores", "Osmond", "Karaugh", "Wilbur", "Kasandra", "Renae", "Kaety", "Dora", "Gaye", "Amaryllis",
+            "Katelyn", "Dacre", "Prudence", "Ebony", "Camron", "Jerrold", "Vivyan", "Randall", "Donna", "Misty",
+            "Damon", "Selby", "Esmund", "Rian", "Garry", "Julius", "Raelene", "Clement", "Dom", "Tibby", "Moss",
+            "Millicent", "Gwendoline", "Berry", "Ashleigh", "Lilac", "Quin", "Vere", "Creighton", "Harriet", "Malvina",
+            "Lianne", "Pearle", "Kizzie", "Kara", "Petula", "Jeanie", "Maria", "Pacey", "Victoria", "Huey", "Toni",
+            "Rose", "Wallis", "Diggory", "Josiah", "Delma", "Keysha", "Channing", "Prue", "Lee", "Ryan", "Sidney",
+            "Valerie", "Clancy", "Ezra", "Gilbert", "Clare", "Laz", "Crofton", "Mike", "Annabella", "Tara", "Eldred",
+            "Arthur", "Jaylon", "Peronel", "Paden", "Dot", "Marian", "Amyas", "Alexus", "Esmond", "Abbie", "Stanley",
+            "Brittani", "Vickie", "Errol", "Kimberlee", "Uland", "Ebenezer", "Howie", "Eveline", "Andrea", "Trish",
+            "Hopkin", "Bryanna", "Temperance", "Valarie", "Femie", "Alix", "Terrell", "Lewin", "Lorrin", "Happy",
+            "Micah", "Rachyl", "Sloan", "Gertrude", "Elizabeth", "Dorris", "Andra", "Bram", "Gary", "Jeannine",
+            "Maurene", "Irene", "Yolonda", "Jonty", "Coleen", "Cecelia", "Chantal", "Stuart", "Caris", "Ros",
+            "Kaleigh", "Mirabelle", "Kolby", "Primrose", "Susannah", "Ginny", "Jinny", "Dolly", "Lettice", "Sonny",
+            "Melva", "Ernest", "Garret", "Reagan", "Trenton", "Gallagher", "Edwin", "Nikolas", "Corrie", "Lynette",
+            "Ettie", "Sly", "Debbi", "Eudora", "Brittney", "Tacey", "Marius", "Anima", "Gordon", "Olivia", "Kortney",
+            "Shantel", "Kolleen", "Nevaeh", "Buck", "Sera", "Liliana", "Aric", "Kalyn", "Mick", "Libby", "Ingram",
+            "Alexandria", "Darleen", "Jacklyn", "Hughie", "Tyler", "Aida", "Ronda", "Deemer", "Taryn", "Laureen",
+            "Samantha", "Dave", "Hardy", "Baldric", "Montgomery", "Gus", "Ellis", "Titania", "Luke", "Chase", "Haidee",
+            "Mayra", "Isabell", "Trinity", "Milo", "Abigail", "Tacita", "Meg", "Hervey", "Natasha", "Sadie", "Holden",
+            "Dee", "Mansel", "Perry", "Randi", "Frederica", "Georgina", "Kolour", "Debbie", "Seraphina", "Elspet",
+            "Julyan", "Raven", "Zavia", "Jarvis", "Jaymes", "Grover", "Cairo", "Alea", "Jordon", "Braxton", "Donny",
+            "Rhoda", "Tonya", "Bee", "Alyssia", "Ashlyn", "Reanna", "Lonny", "Arlene", "Deb", "Jane", "Nikole",
+            "Bettina", "Harrison", "Tamzen", "Arielle", "Adelaide", "Faith", "Bridie", "Wilburn", "Fern", "Nan",
+            "Shaw", "Zeke", "Alan", "Dene", "Gina", "Alexa", "Bailey", "Sal", "Tammy", "Maximillian", "America",
+            "Sylvana", "Fitz", "Mo", "Marissa", "Cass", "Eldon", "Wilfrid", "Tel", "Joann", "Kendra", "Tolly",
+            "Leanne", "Ferdie", "Haven", "Lucas", "Marlee", "Cyrilla", "Red", "Phoenix", "Jazmin", "Carin", "Gena",
+            "Lashonda", "Tucker", "Genette", "Kizzy", "Winifred", "Melody", "Keely", "Kaylyn", "Radcliff", "Lettie",
+            "Foster", "Lyndsey", "Nicholas", "Farley", "Louisa", "Dana", "Dortha", "Francine", "Doran", "Bonita",
+            "Hal", "Sawyer", "Reginald", "Aislin", "Nathan", "Baylee", "Abilene", "Ladonna", "Maurine", "Shelly",
+            "Deandre", "Jasmin", "Roderic", "Tiffany", "Amanda", "Verity", "Wilford", "Gayelord", "Whitney", "Demelza",
+            "Kenton", "Alberta", "Kyra", "Tabitha", "Sampson", "Korey", "Lillian", "Edison", "Clayton", "Steph",
+            "Maya", "Dusty", "Jim", "Ronny", "Adrianne", "Bernard", "Harris", "Kiley", "Alexander", "Kisha", "Ethalyn",
+            "Patience", "Briony", "Indigo", "Aureole", "Makenzie", "Molly", "Sherilyn", "Barry", "Laverne", "Hunter",
+            "Rocky", "Tyreek", "Madalyn", "Phyliss", "Chet", "Beatrice", "Faye", "Lavina", "Madelyn", "Tracey",
+            "Gyles", "Patti", "Carlyn", "Stephanie", "Jackalyn", "Larrie", "Kimmy", "Isolda", "Emelina", "Lis",
+            "Zillah", "Cody", "Sheard", "Rufus", "Paget", "Mae", "Rexanne", "Luvinia", "Tamsen", "Rosanna", "Greig",
+            "Stacia", "Mabelle", "Quianna", "Lotus", "Delice", "Bradford", "Angus", "Cosmo", "Earlene", "Adrian",
+            "Arlie", "Noelle", "Sabella", "Isa", "Adelle", "Innocent", "Kirby", "Trixie", "Kenelm", "Nelda", "Melia",
+            "Kendal", "Dorinda", "Placid", "Linette", "Kam", "Sherisse", "Evan", "Ewart", "Janice", "Linton",
+            "Jacaline", "Charissa", "Douglas", "Aileen", "Kemp", "Oli", "Amethyst", "Rosie", "Nigella", "Sherill",
+            "Anderson", "Alanna", "Eric", "Claudia", "Jennifer", "Boniface", "Harriet", "Vernon", "Lucy", "Shawnee",
+            "Gerard", "Cecily", "Romey", "Randall", "Wade", "Lux", "Dawson", "Gregg", "Kade", "Roxanne", "Melinda",
+            "Rolland", "Rowanne", "Fannie", "Isidore", "Melia", "Harvie", "Salal", "Eleonor", "Jacquette", "Lavone",
+            "Shanika", "Tarquin", "Janet", "Josslyn", "Maegan", "Augusta", "Aubree", "Francene", "Martie", "Marisa",
+            "Tyreek", "Tatianna", "Caleb", "Sheridan", "Nellie", "Barbara", "Wat", "Jayla", "Esmaralda", "Graeme",
+            "Lavena", "Jemima", "Nikolas", "Triston", "Portia", "Kyla", "Marcus", "Raeburn", "Jamison", "Earl", "Wren",
+            "Leighton", "Lagina", "Lucasta", "Dina", "Amaranta", "Jessika", "Claud", "Bernard", "Winifred", "Ebba",
+            "Sammi", "Gall", "Chloe", "Ottoline", "Herbert", "Janice", "Gareth", "Channing", "Caleigh", "Kailee",
+            "Ralphie", "Tamzen", "Quincy", "Beaumont", "Albert", "Jadyn", "Violet", "Luanna", "Moriah", "Humbert",
+            "Jed", "Leona", "Hale", "Mitch", "Marlin", "Nivek", "Darwin", "Dirk", "Liliana", "Meadow", "Bernadine",
+            "Jorie", "Peyton", "Astra", "Roscoe", "Gina", "Lovell", "Jewel", "Romayne", "Rosy", "Imogene",
+            "Margaretta", "Lorinda", "Hopkin", "Bobby", "Flossie", "Bennie", "Horatio", "Jonah", "Lyn", "Deana",
+            "Juliana", "Blanch", "Wright", "Kendal", "Woodrow", "Tania", "Austyn", "Val", "Mona", "Charla", "Rudyard",
+            "Pamela", "Raven", "Zena", "Nicola", "Kaelea", "Conor", "Virgil", "Sonnie", "Goodwin", "Christianne",
+            "Linford", "Myron", "Denton", "Charita", "Brody", "Ginnie", "Harrison", "Jeanine", "Quin", "Isolda",
+            "Zoie", "Pearce", "Margie", "Larrie", "Angelina", "Marcia", "Jessamine", "Delilah", "Dick", "Luana",
+            "Delicia", "Lake", "Luvenia", "Vaughan", "Concordia", "Gayelord", "Cheyenne", "Felix", "Dorris", "Pen",
+            "Kristeen", "Parris", "Everitt", "Josephina", "Amy", "Tommie", "Adrian", "April", "Rosaline", "Zachery",
+            "Trace", "Phoebe", "Jenelle", "Kameron", "Katharine", "Media", "Colton", "Tad", "Quianna", "Kerenza",
+            "Greta", "Luvinia", "Pete", "Tonya", "Beckah", "Barbra", "Jon", "Tetty", "Corey", "Sylvana", "Kizzy",
+            "Korey", "Trey", "Haydee", "Penny", "Mandy", "Panda", "Coline", "Ramsey", "Sukie", "Annabel", "Sarina",
+            "Corbin", "Suzanna", "Rob", "Duana", "Shell", "Jason", "Eddy", "Rube", "Roseann", "Celia", "Brianne",
+            "Nerissa", "Jera", "Humphry", "Ashlynn", "Terrence", "Philippina", "Coreen", "Kolour", "Indiana", "Paget",
+            "Marlyn", "Hester", "Isbel", "Ocean", "Harris", "Leslie", "Vere", "Monroe", "Isabelle", "Bertie", "Clitus",
+            "Dave", "Alethea", "Lessie", "Louiza", "Madlyn", "Garland", "Wolf", "Lalo", "Donny", "Amabel", "Tianna",
+            "Louie", "Susie", "Mackenzie", "Renie", "Tess", "Marmaduke", "Gwendolen", "Bettina", "Beatrix", "Esmund",
+            "Minnie", "Carlie", "Barnabas", "Ruthie", "Honour", "Haylie", "Xavior", "Freddie", "Ericka", "Aretha",
+            "Edie", "Madelina", "Anson", "Tabby", "Derrick", "Jocosa", "Deirdre", "Aislin", "Chastity", "Abigail",
+            "Wynonna", "Zo", "Eldon", "Krystine", "Ghislaine", "Zavia", "Nolene", "Marigold", "Kelley", "Sylvester",
+            "Odell", "George", "Laurene", "Franklyn", "Clarice", "Mo", "Dustin", "Debbi", "Lina", "Tony", "Acacia",
+            "Hettie", "Natalee", "Marcie", "Brittany", "Elnora", "Rachel", "Dawn", "Basil", "Christal", "Anjelica",
+            "Fran", "Tawny", "Delroy", "Tameka", "Lillie", "Ceara", "Deanna", "Deshaun", "Ken", "Bradford", "Justina",
+            "Merle", "Draven", "Gretta", "Harriette", "Webster", "Nathaniel", "Anemone", "Coleen", "Ruth", "Chryssa",
+            "Hortensia", "Saffie", "Deonne", "Leopold", "Harlan", "Lea", "Eppie", "Lucinda", "Tilda", "Fanny", "Titty",
+            "Lockie", "Jepson", "Sherisse", "Maralyn", "Ethel", "Sly", "Ebenezer", "Canute", "Ella", "Freeman",
+            "Reuben", "Olivette", "Nona", "Rik", "Amice", "Kristine", "Kathie", "Jayne", "Jeri", "Mckenna", "Bertram",
+            "Kaylee", "Livia", "Gil", "Wallace", "Maryann", "Keeleigh", "Laurinda", "Doran", "Khloe", "Dakota",
+            "Yaron", "Kimberleigh", "Gytha", "Doris", "Marylyn", "Benton", "Linnette", "Esther", "Jakki", "Rowina",
+            "Marian", "Roselyn", "Norbert", "Maggie", "Caesar", "Phinehas", "Jerry", "Jasmine", "Antonette", "Miriam",
+            "Monna", "Maryvonne", "Jacquetta", "Bernetta", "Napier", "Annie", "Gladwin", "Sheldon", "Aric", "Elouise",
+            "Gawain", "Kristia", "Gabe", "Kyra", "Red", "Tod", "Dudley", "Lorraine", "Ryley", "Sabina", "Poppy",
+            "Leland", "Aileen", "Eglantine", "Alicia", "Jeni", "Addy", "Tiffany", "Geffrey", "Lavina", "Collin",
+            "Clover", "Vin", "Jerome", "Doug", "Vincent", "Florence", "Scarlet", "Celeste", "Desdemona", "Tiphanie",
+            "Kassandra", "Ashton", "Madison", "Art", "Magdalene", "Iona", "Josepha", "Anise", "Ferne", "Derek",
+            "Huffie", "Qiana", "Ysabel", "Tami", "Shannah", "Xavier", "Willard", "Winthrop", "Vickie", "Maura",
+            "Placid", "Tiara", "Reggie", "Elissa", "Isa", "Chrysanta", "Jeff", "Bessie", "Terri", "Amilia", "Brett",
+            "Daniella", "Damion", "Carolina", "Maximillian", "Travers", "Benjamin", "Oprah", "Darcy", "Yolanda",
+            "Nicolina", "Crofton", "Jarrett", "Kaitlin", "Shauna", "Keren", "Bevis", "Kalysta", "Sharron", "Alyssa",
+            "Blythe", "Zelma", "Caelie", "Norwood", "Billie", "Patrick", "Gary", "Cambria", "Tylar", "Mason", "Helen",
+            "Melyssa", "Gene", "Gilberta", "Carter", "Herbie", "Harmonie", "Leola", "Eugenia", "Clint", "Pauletta",
+            "Edwyna", "Georgina", "Teal", "Harper", "Izzy", "Dillon", "Kezia", "Evangeline", "Colene", "Madelaine",
+            "Zilla", "Rudy", "Dottie", "Caris", "Morton", "Marge", "Tacey", "Parker", "Troy", "Liza", "Lewin",
+            "Tracie", "Justine", "Dallas", "Linden", "Ray", "Loretta", "Teri", "Elvis", "Diane", "Julianna", "Manfred",
+            "Denise", "Eireen", "Ann", "Kenith", "Linwood", "Kathlyn", "Bernice", "Shelley", "Oswald", "Amedeus",
+            "Homer", "Tanzi", "Ted", "Ralphina", "Hyacinth", "Lotus", "Matthias", "Arlette", "Clark", "Cecil",
+            "Elspeth", "Alvena", "Noah", "Millard", "Brenden", "Cole", "Philipa", "Nina", "Thelma", "Iantha", "Reid",
+            "Jefferson", "Meg", "Elsie", "Shirlee", "Nathan", "Nancy", "Simona", "Racheal", "Carin", "Emory", "Delice",
+            "Kristi", "Karaugh", "Kaety", "Tilly", "Em", "Alanis", "Darrin", "Jerrie", "Hollis", "Cary", "Marly",
+            "Carita", "Jody", "Farley", "Hervey", "Rosalin", "Cuthbert", "Stewart", "Jodene", "Caileigh", "Briscoe",
+            "Dolores", "Sheree", "Eustace", "Nigel", "Detta", "Barret", "Rowland", "Kenny", "Githa", "Zoey", "Adela",
+            "Petronella", "Opal", "Coleman", "Niles", "Cyril", "Dona", "Alberic", "Allannah", "Jules", "Avalon",
+            "Hadley", "Thomas", "Renita", "Calanthe", "Heron", "Shawnda", "Chet", "Malina", "Manny", "Rina", "Frieda",
+            "Eveleen", "Deshawn", "Amos", "Raelene", "Paige", "Molly", "Nannie", "Ileen", "Brendon", "Milford",
+            "Unice", "Rebeccah", "Caedmon", "Gae", "Doreen", "Vivian", "Louis", "Raphael", "Vergil", "Lise", "Glenn",
+            "Karyn", "Terance", "Reina", "Jake", "Gordon", "Wisdom", "Isiah", "Gervase", "Fern", "Marylou", "Roddy",
+            "Justy", "Derick", "Shantelle", "Adam", "Chantel", "Madoline", "Emmerson", "Lexie", "Mickey", "Stephen",
+            "Dane", "Stacee", "Elwin", "Tracey", "Alexandra", "Ricky", "Ian", "Kasey", "Rita", "Alanna", "Georgene",
+            "Deon", "Zavier", "Ophelia", "Deforest", "Lowell", "Zubin", "Hardy", "Osmund", "Tabatha", "Debby",
+            "Katlyn", "Tallulah", "Priscilla", "Braden", "Wil", "Keziah", "Jen", "Aggie", "Korbin", "Lemoine",
+            "Barnaby", "Tranter", "Goldie", "Roderick", "Trina", "Emery", "Pris", "Sidony", "Adelle", "Tate", "Wilf",
+            "Zola", "Brande", "Chris", "Calanthia", "Lilly", "Kaycee", "Lashonda", "Jasmin", "Elijah", "Shantel",
+            "Simon", "Rosalind", "Jarod", "Kaylie", "Corrine", "Joselyn", "Archibald", "Mariabella", "Winton",
+            "Merlin", "Chad", "Ursula", "Kristopher", "Hewie", "Adrianna", "Lyndsay", "Jasmyn", "Tim", "Evette",
+            "Margaret", "Samson", "Bronte", "Terence", "Leila", "Candice", "Tori", "Jamey", "Coriander", "Conrad",
+            "Floyd", "Karen", "Lorin", "Maximilian", "Cairo", "Emily", "Yasmin", "Karolyn", "Bryan", "Lanny",
+            "Kimberly", "Rick", "Chaz", "Krystle", "Lyric", "Laura", "Garrick", "Flip", "Monty", "Brendan",
+            "Ermintrude", "Rayner", "Merla", "Titus", "Marva", "Patricia", "Leone", "Tracy", "Jaqueline", "Hallam",
+            "Delores", "Cressida", "Carlyle", "Leann", "Kelcey", "Laurence", "Ryan", "Reynold", "Mark", "Collyn",
+            "Audie", "Sammy", "Ellery", "Sallie", "Pamelia", "Adolph", "Lydia", "Titania", "Ron", "Bridger", "Aline",
+            "Read", "Kelleigh", "Weldon", "Irving", "Garey", "Diggory", "Evander", "Kylee", "Deidre", "Ormond",
+            "Laurine", "Reannon", "Arline", "Pat"
+
+    };
+
+    public static String[] jargon = { "wireless", "signal", "network", "3G", "plan", "touch-screen",
+            "customer-service", "reachability", "voice-command", "shortcut-menu", "customization", "platform", "speed",
+            "voice-clarity", "voicemail-service" };
+
+    public static String[] vendors = { "at&t", "verizon", "t-mobile", "sprint", "motorola", "samsung", "iphone" };
+
+    public static String[] org_list = { "Latsonity", "ganjalax", "Zuncan", "Lexitechno", "Hot-tech", "subtam",
+            "Coneflex", "Ganjatax", "physcane", "Tranzap", "Qvohouse", "Zununoing", "jaydax", "Keytech", "goldendexon",
+            "Villa-tech", "Trustbam", "Newcom", "Voltlane", "Ontohothex", "Ranhotfan", "Alphadax", "Transhigh",
+            "kin-ron", "Doublezone", "Solophase", "Vivaace", "silfind", "Basecone", "sonstreet", "Freshfix",
+            "Techitechi", "Kanelectrics", "linedexon", "Goldcity", "Newfase", "Technohow", "Zimcone", "Salthex",
+            "U-ron", "Solfix", "whitestreet", "Xx-technology", "Hexviafind", "over-it", "Strongtone", "Tripplelane",
+            "geomedia", "Scotcity", "Inchex", "Vaiatech", "Striptaxon", "Hatcom", "tresline", "Sanjodax", "freshdox",
+            "Sumlane", "Quadlane", "Newphase", "overtech", "Voltbam", "Icerunin", "Fixdintex", "Hexsanhex", "Statcode",
+            "Greencare", "U-electrics", "Zamcorporation", "Ontotanin", "Tanzimcare", "Groovetex", "Ganjastrip",
+            "Redelectronics", "Dandamace", "Whitemedia", "strongex", "Streettax", "highfax", "Mathtech", "Xx-drill",
+            "Sublamdox", "Unijobam", "Rungozoom", "Fixelectrics", "Villa-dox", "Ransaofan", "Plexlane", "itlab",
+            "Lexicone", "Fax-fax", "Viatechi", "Inchdox", "Kongreen", "Doncare", "Y-geohex", "Opeelectronics",
+            "Medflex", "Dancode", "Roundhex", "Labzatron", "Newhotplus", "Sancone", "Ronholdings", "Quoline",
+            "zoomplus", "Fix-touch", "Codetechno", "Tanzumbam", "Indiex", "Canline" };
+}
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/DataGeneratorDriver.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/DataGeneratorDriver.java
new file mode 100644
index 0000000..b2ba88e
--- /dev/null
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/DataGeneratorDriver.java
@@ -0,0 +1,18 @@
+package edu.uci.ics.asterix.tools.external.data;
+
+import java.util.Iterator;
+
+import edu.uci.ics.asterix.tools.external.data.DataGenerator.InitializationInfo;
+import edu.uci.ics.asterix.tools.external.data.DataGenerator.TweetMessage;
+
+public class DataGeneratorDriver {
+
+    public static void main(String[] args) {
+
+        DataGenerator.initialize(new InitializationInfo());
+        Iterator<TweetMessage> tweetIterator = DataGenerator.getTwitterMessageIterator();
+        while (tweetIterator.hasNext()) {
+            System.out.println(tweetIterator.next().toString());
+        }
+    }
+}
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/RateControlledFileSystemBasedAdapter.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/RateControlledFileSystemBasedAdapter.java
index 49098a4..32b539c 100644
--- a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/RateControlledFileSystemBasedAdapter.java
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/RateControlledFileSystemBasedAdapter.java
@@ -20,8 +20,8 @@
 
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
 import edu.uci.ics.asterix.external.dataset.adapter.FileSystemBasedAdapter;
-import edu.uci.ics.asterix.external.dataset.adapter.ITypedDatasourceAdapter;
-import edu.uci.ics.asterix.feed.managed.adapter.IManagedFeedAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IManagedFeedAdapter;
+import edu.uci.ics.asterix.metadata.feeds.ITypedDatasourceAdapter;
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.ATypeTag;
 import edu.uci.ics.asterix.runtime.operators.file.ADMDataParser;
@@ -39,9 +39,11 @@
 import edu.uci.ics.hyracks.dataflow.std.file.ITupleParserFactory;
 
 /**
- * An adapter that simulates a feed from the contents of a source file. The file can be on the local file
- * system or on HDFS. The feed ends when the content of the source file has been ingested.
+ * An adapter that simulates a feed from the contents of a source file. The file
+ * can be on the local file system or on HDFS. The feed ends when the content of
+ * the source file has been ingested.
  */
+
 public class RateControlledFileSystemBasedAdapter extends FileSystemBasedAdapter implements ITypedDatasourceAdapter,
         IManagedFeedAdapter {
 
@@ -133,8 +135,8 @@
     }
 
     @Override
-    public void alter(Map<String, String> properties) {
-        ((RateControlledTupleParser) parser).setInterTupleInterval(Long.parseLong(properties
+    public void alter(Map<String, Object> properties) {
+        ((RateControlledTupleParser) parser).setInterTupleInterval(Long.parseLong((String) properties
                 .get(RateControlledTupleParser.INTER_TUPLE_INTERVAL)));
     }
 
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/RateControlledFileSystemBasedAdapterFactory.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/RateControlledFileSystemBasedAdapterFactory.java
index bf3c086..ff3108f 100644
--- a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/RateControlledFileSystemBasedAdapterFactory.java
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/RateControlledFileSystemBasedAdapterFactory.java
@@ -17,9 +17,9 @@
 import java.util.Map;
 
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
-import edu.uci.ics.asterix.external.adapter.factory.IGenericDatasetAdapterFactory;
 import edu.uci.ics.asterix.external.dataset.adapter.FileSystemBasedAdapter;
-import edu.uci.ics.asterix.external.dataset.adapter.IDatasourceAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IDatasourceAdapter;
+import edu.uci.ics.asterix.metadata.feeds.IGenericDatasetAdapterFactory;
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.IAType;
 
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/SyntheticTwitterFeedAdapter.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/SyntheticTwitterFeedAdapter.java
new file mode 100644
index 0000000..33fa168
--- /dev/null
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/SyntheticTwitterFeedAdapter.java
@@ -0,0 +1,231 @@
+package edu.uci.ics.asterix.tools.external.data;
+
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Logger;
+
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.dataflow.data.nontagged.serde.ARecordSerializerDeserializer;
+import edu.uci.ics.asterix.external.dataset.adapter.IPullBasedFeedClient;
+import edu.uci.ics.asterix.external.dataset.adapter.PullBasedAdapter;
+import edu.uci.ics.asterix.external.dataset.adapter.PullBasedFeedClient;
+import edu.uci.ics.asterix.om.base.AMutableDateTime;
+import edu.uci.ics.asterix.om.base.AMutableInt32;
+import edu.uci.ics.asterix.om.base.AMutablePoint;
+import edu.uci.ics.asterix.om.base.AMutableRecord;
+import edu.uci.ics.asterix.om.base.AMutableString;
+import edu.uci.ics.asterix.om.base.AMutableUnorderedList;
+import edu.uci.ics.asterix.om.base.IAObject;
+import edu.uci.ics.asterix.om.types.ARecordType;
+import edu.uci.ics.asterix.om.types.AUnorderedListType;
+import edu.uci.ics.asterix.om.types.BuiltinType;
+import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.asterix.tools.external.data.DataGenerator.InitializationInfo;
+import edu.uci.ics.asterix.tools.external.data.DataGenerator.Message;
+import edu.uci.ics.asterix.tools.external.data.DataGenerator.TweetMessage;
+import edu.uci.ics.asterix.tools.external.data.DataGenerator.TweetMessageIterator;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+
+/**
+ * TPS can be configured between 1 and 20,000  
+ * @author ramang
+ *
+ */
+public class SyntheticTwitterFeedAdapter extends PullBasedAdapter {
+
+    private static final long serialVersionUID = 1L;
+    private Map<String, Object> configuration;
+
+    public SyntheticTwitterFeedAdapter(Map<String, Object> configuration) throws AsterixException {
+        this.configuration = configuration;
+
+        String[] userFieldNames = new String[] { "screen-name", "lang", "friends_count", "statuses_count", "name",
+                "followers_count" };
+
+        IAType[] userFieldTypes = new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.AINT32,
+                BuiltinType.AINT32, BuiltinType.ASTRING, BuiltinType.AINT32 };
+        ARecordType userRecordType = new ARecordType("TwitterUserType", userFieldNames, userFieldTypes, false);
+
+        String[] fieldNames = new String[] { "tweetid", "user", "sender-location", "send-time", "referred-topics",
+                "message-text" };
+
+        AUnorderedListType unorderedListType = new AUnorderedListType(BuiltinType.ASTRING, "referred-topics");
+        IAType[] fieldTypes = new IAType[] { BuiltinType.ASTRING, userRecordType, BuiltinType.APOINT,
+                BuiltinType.ADATETIME, unorderedListType, BuiltinType.ASTRING };
+        adapterOutputType = new ARecordType("TweetMessageType", fieldNames, fieldTypes, false);
+    }
+
+    @Override
+    public AdapterType getAdapterType() {
+        return AdapterType.READ;
+    }
+
+    @Override
+    public void configure(Map<String, Object> configuration) throws Exception {
+        this.configuration = configuration;
+
+    }
+
+    @Override
+    public void initialize(IHyracksTaskContext ctx) throws Exception {
+        this.ctx = ctx;
+    }
+
+    @Override
+    public IPullBasedFeedClient getFeedClient(int partition) throws Exception {
+        return new SyntheticTwitterFeedClient(configuration, adapterOutputType, partition);
+    }
+
+    @Override
+    public AlgebricksPartitionConstraint getPartitionConstraint() throws Exception {
+        return new AlgebricksCountPartitionConstraint(1);
+    }
+
+    private static class SyntheticTwitterFeedClient extends PullBasedFeedClient implements IPullBasedFeedClient {
+
+        private static final Logger LOGGER = Logger.getLogger(SyntheticTwitterFeedClient.class.getName());
+
+        public static final String KEY_DURATION = "duration";
+        public static final String KEY_TPS = "tps";
+        public static final String KEY_EXCEPTION_PERIOD = "exception-period";
+
+        private int duration;
+        private long tweetInterval;
+        private int numTweetsBeforeDelay;
+        private TweetMessageIterator tweetIterator = null;
+        private long exeptionInterval;
+
+        private IAObject[] mutableFields;
+        private ARecordType outputRecordType;
+        private int partition;
+        private int tweetCount = 0;
+
+        public SyntheticTwitterFeedClient(Map<String, Object> configuration, ARecordType outputRecordType,
+                int partition) throws AsterixException {
+            this.outputRecordType = outputRecordType;
+            String value = (String)configuration.get(KEY_DURATION);
+            duration = value != null ? Integer.parseInt(value) : 60;
+            initializeTweetRate((String)configuration.get(KEY_TPS));
+            InitializationInfo info = new InitializationInfo();
+            info.timeDurationInSecs = duration;
+            DataGenerator.initialize(info);
+            tweetIterator = new TweetMessageIterator(duration);
+            initialize();
+        }
+
+        private void initializeTweetRate(String tps) {
+            numTweetsBeforeDelay = 0;
+            if (tps == null) {
+                tweetInterval = 0;
+            } else {
+                int val = Integer.parseInt(tps);
+                double interval = new Double(((double) 1000 / val));
+                if (interval > 1) {
+                    tweetInterval = (long) interval;
+                    numTweetsBeforeDelay = 1;
+                } else {
+                    tweetInterval = 1;
+                    Double numTweets = new Double(1 / interval);
+                    if (numTweets.intValue() != numTweets) {
+                        tweetInterval = 10;
+                        numTweetsBeforeDelay = (new Double(10 * numTweets * 1.4)).intValue();
+                    } else {
+                        numTweetsBeforeDelay = new Double((numTweets * 1.4)).intValue();
+                    }
+                }
+            }
+
+        }
+
+        private void writeTweet(TweetMessage next) {
+
+            //tweet id
+            ((AMutableString) mutableFields[0]).setValue(next.getTweetid());
+            mutableRecord.setValueAtPos(0, mutableFields[0]);
+
+            // user 
+            AMutableRecord userRecord = ((AMutableRecord) mutableFields[1]);
+            ((AMutableString) userRecord.getValueByPos(0)).setValue(next.getUser().getScreenName());
+            ((AMutableString) userRecord.getValueByPos(1)).setValue("en");
+            ((AMutableInt32) userRecord.getValueByPos(2)).setValue(next.getUser().getFriendsCount());
+            ((AMutableInt32) userRecord.getValueByPos(3)).setValue(next.getUser().getStatusesCount());
+            ((AMutableString) userRecord.getValueByPos(4)).setValue(next.getUser().getName());
+            ((AMutableInt32) userRecord.getValueByPos(5)).setValue(next.getUser().getFollowersCount());
+            mutableRecord.setValueAtPos(1, userRecord);
+
+            // location
+            ((AMutablePoint) mutableFields[2]).setValue(next.getSenderLocation().getLatitude(), next
+                    .getSenderLocation().getLongitude());
+            mutableRecord.setValueAtPos(2, mutableFields[2]);
+
+            // time
+            ((AMutableDateTime) mutableFields[3]).setValue(next.getSendTime().getChrononTime());
+            mutableRecord.setValueAtPos(3, mutableFields[3]);
+
+            // referred topics
+            ((AMutableUnorderedList) mutableFields[4]).clear();
+            List<String> referredTopics = next.getReferredTopics();
+            for (String topic : referredTopics) {
+                ((AMutableUnorderedList) mutableFields[4]).add(new AMutableString(topic));
+            }
+            mutableRecord.setValueAtPos(4, mutableFields[4]);
+
+            // text
+            Message m = next.getMessageText();
+            char[] content = m.getMessage();
+            ((AMutableString) mutableFields[5]).setValue(new String(content, 0, m.getLength()));
+            mutableRecord.setValueAtPos(5, mutableFields[5]);
+
+        }
+
+        @Override
+        public void resetOnFailure(Exception e) throws AsterixException {
+            // TODO Auto-generated method stub
+
+        }
+
+        @Override
+        public boolean alter(Map<String, Object> configuration) {
+            // TODO Auto-generated method stub
+            return false;
+        }
+
+        @Override
+        public InflowState setNextRecord() throws Exception {
+            boolean moreData = tweetIterator.hasNext();
+            if (!moreData) {
+                return InflowState.NO_MORE_DATA;
+            }
+            writeTweet(tweetIterator.next());
+            if (tweetInterval != 0) {
+                tweetCount++;
+                if (tweetCount == numTweetsBeforeDelay) {
+                    Thread.sleep(tweetInterval);
+                    tweetCount = 0;
+                }
+            }
+            return InflowState.DATA_AVAILABLE;
+        }
+
+        private void initialize() throws AsterixException {
+            ARecordType userRecordType = (ARecordType) outputRecordType.getFieldTypes()[1];
+            IAObject[] userMutableFields = new IAObject[] { new AMutableString(""), new AMutableString(""),
+                    new AMutableInt32(0), new AMutableInt32(0), new AMutableString(""), new AMutableInt32(0) };
+            AUnorderedListType unorderedListType = new AUnorderedListType(BuiltinType.ASTRING, "referred-topics");
+            mutableFields = new IAObject[] { new AMutableString(""),
+                    new AMutableRecord(userRecordType, userMutableFields), new AMutablePoint(0, 0),
+                    new AMutableDateTime(0), new AMutableUnorderedList(unorderedListType), new AMutableString("") };
+            recordSerDe = new ARecordSerializerDeserializer(outputRecordType);
+            mutableRecord = new AMutableRecord(outputRecordType, mutableFields);
+
+        }
+
+        @Override
+        public void stop() {
+            // TODO Auto-generated method stub
+
+        }
+    }
+}
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/SyntheticTwitterFeedAdapterFactory.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/SyntheticTwitterFeedAdapterFactory.java
new file mode 100644
index 0000000..7832f11
--- /dev/null
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/SyntheticTwitterFeedAdapterFactory.java
@@ -0,0 +1,45 @@
+/*
+x * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.tools.external.data;
+
+import java.util.Map;
+
+import edu.uci.ics.asterix.metadata.feeds.IDatasourceAdapter;
+import edu.uci.ics.asterix.metadata.feeds.ITypedDatasetAdapterFactory;
+
+/**
+ * Factory class for creating @see{RateControllerFileSystemBasedAdapter} The
+ * adapter simulates a feed from the contents of a source file. The file can be
+ * on the local file system or on HDFS. The feed ends when the content of the
+ * source file has been ingested.
+ */
+public class SyntheticTwitterFeedAdapterFactory implements ITypedDatasetAdapterFactory {
+
+    /**
+     * 
+     */
+    private static final long serialVersionUID = 1L;
+
+    @Override
+    public String getName() {
+        return "synthetic_twitter_feed";
+    }
+
+    @Override
+    public IDatasourceAdapter createAdapter(Map<String, Object> configuration) throws Exception {
+        return new SyntheticTwitterFeedAdapter(configuration);
+    }
+
+}
\ No newline at end of file
diff --git a/asterix-tools/src/main/resources/test.properties b/asterix-tools/src/main/resources/test.properties
old mode 100755
new mode 100644
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/locking/LockManager.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/locking/LockManager.java
index a18b3d3..b09cd52 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/locking/LockManager.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/locking/LockManager.java
@@ -32,9 +32,9 @@
 import edu.uci.ics.asterix.common.transactions.ITransactionContext;
 import edu.uci.ics.asterix.common.transactions.ITransactionManager.TransactionState;
 import edu.uci.ics.asterix.common.transactions.JobId;
+import edu.uci.ics.asterix.common.transactions.LogUtil;
 import edu.uci.ics.asterix.common.transactions.LogicalLogLocator;
 import edu.uci.ics.asterix.transaction.management.service.logging.LogType;
-import edu.uci.ics.asterix.transaction.management.service.logging.LogUtil;
 import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionManagementConstants.LockManagerConstants.LockMode;
 import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionSubsystem;
 import edu.uci.ics.hyracks.api.lifecycle.ILifeCycleComponent;
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/IndexLogger.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/IndexLogger.java
index 9c9439c..755e85b 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/IndexLogger.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/IndexLogger.java
@@ -24,6 +24,7 @@
 import edu.uci.ics.asterix.common.transactions.IResourceManager.ResourceType;
 import edu.uci.ics.asterix.common.transactions.ITransactionContext;
 import edu.uci.ics.asterix.common.transactions.ITransactionSubsystem;
+import edu.uci.ics.asterix.common.transactions.LogUtil;
 import edu.uci.ics.asterix.common.transactions.LogicalLogLocator;
 import edu.uci.ics.asterix.common.transactions.ReusableLogContentObject;
 import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogCursor.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogCursor.java
index 645b613..745abb3 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogCursor.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogCursor.java
@@ -24,6 +24,7 @@
 import edu.uci.ics.asterix.common.transactions.ILogCursor;
 import edu.uci.ics.asterix.common.transactions.ILogFilter;
 import edu.uci.ics.asterix.common.transactions.LogManagerProperties;
+import edu.uci.ics.asterix.common.transactions.LogUtil;
 import edu.uci.ics.asterix.common.transactions.LogicalLogLocator;
 import edu.uci.ics.asterix.common.transactions.PhysicalLogLocator;
 
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogManager.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogManager.java
index d4eb8a4..e67f0164 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogManager.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogManager.java
@@ -16,7 +16,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
@@ -25,7 +24,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -45,10 +43,10 @@
 import edu.uci.ics.asterix.common.transactions.ILogger;
 import edu.uci.ics.asterix.common.transactions.ITransactionContext;
 import edu.uci.ics.asterix.common.transactions.LogManagerProperties;
+import edu.uci.ics.asterix.common.transactions.LogUtil;
 import edu.uci.ics.asterix.common.transactions.LogicalLogLocator;
 import edu.uci.ics.asterix.common.transactions.PhysicalLogLocator;
 import edu.uci.ics.asterix.common.transactions.ReusableLogContentObject;
-import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionManagementConstants;
 import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionSubsystem;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
 import edu.uci.ics.hyracks.api.lifecycle.ILifeCycleComponent;
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogUtil.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogUtil.java
deleted file mode 100644
index 035a319..0000000
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogUtil.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.transaction.management.service.logging;
-
-import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import edu.uci.ics.asterix.common.exceptions.ACIDException;
-import edu.uci.ics.asterix.common.transactions.FileUtil;
-import edu.uci.ics.asterix.common.transactions.ILogManager;
-import edu.uci.ics.asterix.common.transactions.LogManagerProperties;
-import edu.uci.ics.asterix.common.transactions.LogicalLogLocator;
-import edu.uci.ics.asterix.common.transactions.PhysicalLogLocator;
-
-
-/**
- * A utility class providing helper methods for the {@link ILogManager}
- */
-public class LogUtil {
-
-    private static final Logger LOGGER = Logger.getLogger(LogUtil.class.getName());
-
-    // read the log directory and initialize log anchor to point to the
-    // current log partition file and the offset where the log manager shall
-    // continue to insert log records.
-
-    public static PhysicalLogLocator initializeLogAnchor(ILogManager logManager) throws ACIDException {
-        int fileId = 0;
-        long offset = 0;
-        LogManagerProperties logManagerProperties = logManager.getLogManagerProperties();
-        File logDir = new File(logManagerProperties.getLogDir());
-        try {
-            if (logDir.exists()) {
-                List<String> logFiles = getLogFiles(logManagerProperties);
-                if (logFiles == null || logFiles.size() == 0) {
-                    FileUtil.createFileIfNotExists(getLogFilePath(logManagerProperties, 0));
-                    if (LOGGER.isLoggable(Level.INFO)) {
-                        LOGGER.info("created a log file: " + getLogFilePath(logManagerProperties, 0));
-                    }
-                } else {
-                    File logFile = new File(LogUtil.getLogFilePath(logManagerProperties,
-                            Long.parseLong(logFiles.get(logFiles.size() - 1))));
-                    fileId = logFiles.size() - 1;
-                    offset = logFile.length();
-                }
-            } else {
-                FileUtil.createNewDirectory(logManagerProperties.getLogDir());
-                if (LOGGER.isLoggable(Level.INFO)) {
-                    LOGGER.info("created the log directory: " + logManagerProperties.getLogDir());
-                }
-                FileUtil.createFileIfNotExists(getLogFilePath(logManagerProperties, 0));
-                if (LOGGER.isLoggable(Level.INFO)) {
-                    LOGGER.info("created a log file: " + getLogFilePath(logManagerProperties, 0));
-                }
-            }
-        } catch (IOException ioe) {
-            throw new ACIDException("Unable to initialize log anchor", ioe);
-        }
-        if (LOGGER.isLoggable(Level.INFO)) {
-            LOGGER.info(" file id :" + fileId + " offset " + offset);
-        }
-        return new PhysicalLogLocator(fileId, offset, logManager);
-    }
-
-    public static List<String> getLogFiles(final LogManagerProperties logManagerProperties) {
-        File logDir = new File(logManagerProperties.getLogDir());
-        String[] logFiles = new String[0];
-        List<String> logFileCollection = new ArrayList<String>();
-        if (logDir.exists()) {
-            logFiles = logDir.list(new FilenameFilter() {
-                public boolean accept(File dir, String name) {
-                    if (name.startsWith(logManagerProperties.getLogFilePrefix())) {
-                        return true;
-                    }
-                    return false;
-                }
-            });
-        }
-        for (String logFile : logFiles) {
-            logFileCollection.add(logFile.substring(logManagerProperties.getLogFilePrefix().length() + 1));
-        }
-        Collections.sort(logFileCollection, new Comparator<String>() {
-            @Override
-            public int compare(String arg0, String arg1) {
-                return Integer.parseInt(arg0) - Integer.parseInt(arg1);
-            }
-        });
-        return logFileCollection;
-    }
-
-    public static long getFileId(String logFilePath, LogManagerProperties logManagerProperties) {
-        String logFileName = logFilePath;
-        if (logFilePath.contains(File.separator)) {
-            logFileName = logFilePath.substring(logFilePath.lastIndexOf(File.separator));
-        }
-        return Long.parseLong(logFileName.substring(logFileName.indexOf(logManagerProperties.getLogFilePrefix())));
-    }
-
-    public static String getLogFilePath(LogManagerProperties logManagerProperties, long fileId) {
-        return logManagerProperties.getLogDir() + File.separator + logManagerProperties.getLogFilePrefix() + "_"
-                + fileId;
-    }
-
-    public static LogicalLogLocator getDummyLogicalLogLocator(ILogManager logManager) {
-        LogicalLogLocator logicalLogLocator = new LogicalLogLocator(-1, null, -1, logManager);
-        return logicalLogLocator;
-    }
-
-    /*
-     * given a lsn, get the offset within the log file where the corresponding
-     * log record is (to be) placed.
-     */
-    public static long getFileOffset(ILogManager logManager, long lsn) {
-        return lsn % logManager.getLogManagerProperties().getLogPartitionSize();
-    }
-
-    /*
-     * given a lsn, get the file id that contains the log record.
-     */
-    public static long getFileId(ILogManager logManager, long lsn) {
-        return lsn / logManager.getLogManagerProperties().getLogPartitionSize();
-    }
-}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/recovery/RecoveryManager.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/recovery/RecoveryManager.java
index 5b946bc..ad1db1f 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/recovery/RecoveryManager.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/recovery/RecoveryManager.java
@@ -49,12 +49,12 @@
 import edu.uci.ics.asterix.common.transactions.IResourceManager;
 import edu.uci.ics.asterix.common.transactions.IResourceManager.ResourceType;
 import edu.uci.ics.asterix.common.transactions.ITransactionContext;
+import edu.uci.ics.asterix.common.transactions.LogUtil;
 import edu.uci.ics.asterix.common.transactions.LogicalLogLocator;
 import edu.uci.ics.asterix.common.transactions.PhysicalLogLocator;
 import edu.uci.ics.asterix.transaction.management.service.logging.IndexResourceManager;
 import edu.uci.ics.asterix.transaction.management.service.logging.LogManager;
 import edu.uci.ics.asterix.transaction.management.service.logging.LogType;
-import edu.uci.ics.asterix.transaction.management.service.logging.LogUtil;
 import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionManagementConstants;
 import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionManager;
 import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionSubsystem;
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/transaction/TransactionContext.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/transaction/TransactionContext.java
index 06a5fbf..33522e3 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/transaction/TransactionContext.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/transaction/TransactionContext.java
@@ -26,8 +26,8 @@
 import edu.uci.ics.asterix.common.transactions.ITransactionContext;
 import edu.uci.ics.asterix.common.transactions.ITransactionManager.TransactionState;
 import edu.uci.ics.asterix.common.transactions.JobId;
+import edu.uci.ics.asterix.common.transactions.LogUtil;
 import edu.uci.ics.asterix.common.transactions.LogicalLogLocator;
-import edu.uci.ics.asterix.transaction.management.service.logging.LogUtil;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
 import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
 import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndex;
diff --git a/asterix-transactions/src/test/java/edu/uci/ics/asterix/transaction/management/logging/test/TransactionWorkloadSimulator.java b/asterix-transactions/src/test/java/edu/uci/ics/asterix/transaction/management/logging/test/TransactionWorkloadSimulator.java
index 1298791..a74b079 100644
--- a/asterix-transactions/src/test/java/edu/uci/ics/asterix/transaction/management/logging/test/TransactionWorkloadSimulator.java
+++ b/asterix-transactions/src/test/java/edu/uci/ics/asterix/transaction/management/logging/test/TransactionWorkloadSimulator.java
@@ -28,11 +28,11 @@
 import edu.uci.ics.asterix.common.transactions.ILogRecordHelper;
 import edu.uci.ics.asterix.common.transactions.IResourceManager;
 import edu.uci.ics.asterix.common.transactions.IResourceManager.ResourceType;
+import edu.uci.ics.asterix.common.transactions.LogUtil;
 import edu.uci.ics.asterix.common.transactions.LogicalLogLocator;
 import edu.uci.ics.asterix.transaction.management.logging.BasicLogger;
 import edu.uci.ics.asterix.transaction.management.service.logging.LogActionType;
 import edu.uci.ics.asterix.transaction.management.service.logging.LogType;
-import edu.uci.ics.asterix.transaction.management.service.logging.LogUtil;
 import edu.uci.ics.asterix.transaction.management.service.transaction.JobIdFactory;
 import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionContext;
 import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionSubsystem;
diff --git a/asterix-transactions/src/test/java/edu/uci/ics/asterix/transaction/management/test/LogRecordReader.java b/asterix-transactions/src/test/java/edu/uci/ics/asterix/transaction/management/test/LogRecordReader.java
index 9aed571..8e13356 100644
--- a/asterix-transactions/src/test/java/edu/uci/ics/asterix/transaction/management/test/LogRecordReader.java
+++ b/asterix-transactions/src/test/java/edu/uci/ics/asterix/transaction/management/test/LogRecordReader.java
@@ -22,10 +22,10 @@
 import edu.uci.ics.asterix.common.transactions.ILogFilter;
 import edu.uci.ics.asterix.common.transactions.ILogManager;
 import edu.uci.ics.asterix.common.transactions.ILogRecordHelper;
+import edu.uci.ics.asterix.common.transactions.LogUtil;
 import edu.uci.ics.asterix.common.transactions.LogicalLogLocator;
 import edu.uci.ics.asterix.common.transactions.PhysicalLogLocator;
 import edu.uci.ics.asterix.transaction.management.service.logging.LogManager;
-import edu.uci.ics.asterix.transaction.management.service.logging.LogUtil;
 import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionSubsystem;
 
 public class LogRecordReader {
diff --git a/asterix-transactions/src/test/java/edu/uci/ics/asterix/transaction/management/test/TransactionSimulator.java b/asterix-transactions/src/test/java/edu/uci/ics/asterix/transaction/management/test/TransactionSimulator.java
index b683a93..4bebbc4 100644
--- a/asterix-transactions/src/test/java/edu/uci/ics/asterix/transaction/management/test/TransactionSimulator.java
+++ b/asterix-transactions/src/test/java/edu/uci/ics/asterix/transaction/management/test/TransactionSimulator.java
@@ -31,10 +31,10 @@
 import edu.uci.ics.asterix.common.transactions.ITransactionContext;
 import edu.uci.ics.asterix.common.transactions.ITransactionManager;
 import edu.uci.ics.asterix.common.transactions.JobId;
+import edu.uci.ics.asterix.common.transactions.LogUtil;
 import edu.uci.ics.asterix.common.transactions.LogicalLogLocator;
 import edu.uci.ics.asterix.transaction.management.logging.IResource;
 import edu.uci.ics.asterix.transaction.management.service.logging.LogType;
-import edu.uci.ics.asterix.transaction.management.service.logging.LogUtil;
 import edu.uci.ics.asterix.transaction.management.service.transaction.JobIdFactory;
 import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionSubsystem;