diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/UnnestToDataScanRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/UnnestToDataScanRule.java
index c087bd1..7a44d72 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/UnnestToDataScanRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/UnnestToDataScanRule.java
@@ -20,6 +20,7 @@
 import org.apache.commons.lang3.mutable.Mutable;
 
 import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.metadata.declared.AqlDataSource;
 import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
 import edu.uci.ics.asterix.metadata.declared.AqlSourceId;
@@ -29,14 +30,10 @@
 import edu.uci.ics.asterix.metadata.entities.Feed;
 import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
 import edu.uci.ics.asterix.metadata.feeds.BuiltinFeedPolicies;
-import edu.uci.ics.asterix.metadata.feeds.FeedConnectionId;
-import edu.uci.ics.asterix.metadata.feeds.FeedUtil;
-import edu.uci.ics.asterix.metadata.feeds.IAdapterFactory;
 import edu.uci.ics.asterix.metadata.utils.DatasetUtils;
 import edu.uci.ics.asterix.om.base.AString;
 import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
 import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.ATypeTag;
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
index c2c75bc..418a83e 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
@@ -30,9 +30,11 @@
 import edu.uci.ics.asterix.common.context.DatasetLifecycleManager;
 import edu.uci.ics.asterix.common.exceptions.ACIDException;
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.common.feeds.IFeedManager;
 import edu.uci.ics.asterix.common.transactions.IAsterixAppRuntimeContextProvider;
 import edu.uci.ics.asterix.common.transactions.ITransactionSubsystem;
 import edu.uci.ics.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
+import edu.uci.ics.asterix.metadata.feeds.FeedManager;
 import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceRepository;
 import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceRepositoryFactory;
 import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionSubsystem;
@@ -89,6 +91,8 @@
     private IIOManager ioManager;
     private boolean isShuttingdown;
 
+    private IFeedManager feedManager;
+
     public AsterixAppRuntimeContext(INCApplicationContext ncApplicationContext) throws AsterixException {
         this.ncApplicationContext = ncApplicationContext;
         compilerProperties = new AsterixCompilerProperties(ASTERIX_PROPERTIES_ACCESSOR);
@@ -138,6 +142,8 @@
                 txnProperties);
         isShuttingdown = false;
 
+        feedManager = new FeedManager(ncApplicationContext.getNodeId());
+
         // The order of registration is important. The buffer cache must registered before recovery and transaction managers.
         LifeCycleComponentManager.INSTANCE.register((ILifeCycleComponent) bufferCache);
         LifeCycleComponentManager.INSTANCE.register((ILifeCycleComponent) indexLifecycleManager);
@@ -238,4 +244,9 @@
         return metadataMergePolicyFactory;
     }
 
+    @Override
+    public IFeedManager getFeedManager() {
+        return feedManager;
+    }
+
 }
\ No newline at end of file
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/FeedDashboardServlet.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/FeedDashboardServlet.java
index f044ac5..5cb53b2 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/FeedDashboardServlet.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/FeedDashboardServlet.java
@@ -31,13 +31,13 @@
 import javax.servlet.http.HttpServletResponse;
 
 import edu.uci.ics.asterix.common.exceptions.ACIDException;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.metadata.MetadataException;
 import edu.uci.ics.asterix.metadata.MetadataManager;
 import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
 import edu.uci.ics.asterix.metadata.entities.FeedActivity;
 import edu.uci.ics.asterix.metadata.entities.FeedActivity.FeedActivityDetails;
 import edu.uci.ics.asterix.metadata.entities.FeedActivity.FeedActivityType;
-import edu.uci.ics.asterix.metadata.feeds.FeedConnectionId;
 
 public class FeedDashboardServlet extends HttpServlet {
     private static final long serialVersionUID = 1L;
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/FeedDataProviderServlet.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/FeedDataProviderServlet.java
index 6985019..437a2c9 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/FeedDataProviderServlet.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/FeedDataProviderServlet.java
@@ -29,13 +29,12 @@
 import org.json.JSONException;
 import org.json.JSONObject;
 
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.hyracks.bootstrap.FeedLifecycleListener;
 import edu.uci.ics.asterix.metadata.MetadataManager;
 import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
 import edu.uci.ics.asterix.metadata.entities.FeedActivity;
 import edu.uci.ics.asterix.metadata.entities.FeedActivity.FeedActivityDetails;
-import edu.uci.ics.asterix.metadata.entities.FeedActivity.FeedActivityType;
-import edu.uci.ics.asterix.metadata.feeds.FeedConnectionId;
 
 public class FeedDataProviderServlet extends HttpServlet {
     private static final long serialVersionUID = 1L;
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/FeedServlet.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/FeedServlet.java
index 546ff4f..96b3c31 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/FeedServlet.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/FeedServlet.java
@@ -29,12 +29,11 @@
 import javax.servlet.http.HttpServletResponse;
 
 import edu.uci.ics.asterix.common.exceptions.ACIDException;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.metadata.MetadataException;
 import edu.uci.ics.asterix.metadata.MetadataManager;
 import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
 import edu.uci.ics.asterix.metadata.entities.FeedActivity;
-import edu.uci.ics.asterix.metadata.entities.FeedActivity.FeedActivityDetails;
-import edu.uci.ics.asterix.metadata.feeds.FeedConnectionId;
 
 public class FeedServlet extends HttpServlet {
     private static final long serialVersionUID = 1L;
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/FeedServletUtil.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/FeedServletUtil.java
index 433f834..ff29a23 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/FeedServletUtil.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/FeedServletUtil.java
@@ -8,8 +8,8 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.hyracks.bootstrap.FeedLifecycleListener;
-import edu.uci.ics.asterix.metadata.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.metadata.feeds.RemoteSocketMessageListener;
 
 public class FeedServletUtil {
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/java/AsterixJavaClient.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/java/AsterixJavaClient.java
index d550689..472eb19 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/java/AsterixJavaClient.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/java/AsterixJavaClient.java
@@ -23,13 +23,13 @@
 import edu.uci.ics.asterix.api.common.Job;
 import edu.uci.ics.asterix.api.common.SessionConfig;
 import edu.uci.ics.asterix.aql.base.Statement;
+import edu.uci.ics.asterix.aql.parser.AQLParser;
+import edu.uci.ics.asterix.aql.parser.ParseException;
 import edu.uci.ics.asterix.aql.translator.AqlTranslator;
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
 import edu.uci.ics.asterix.metadata.MetadataManager;
 import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.asterix.aql.parser.AQLParser;
-import edu.uci.ics.asterix.aql.parser.ParseException;
 
 public class AsterixJavaClient {
     private IHyracksClientConnection hcc;
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
index a18d97c..a0acd44 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
@@ -70,6 +70,7 @@
 import edu.uci.ics.asterix.common.config.GlobalConfig;
 import edu.uci.ics.asterix.common.exceptions.ACIDException;
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.common.functions.FunctionSignature;
 import edu.uci.ics.asterix.file.DatasetOperations;
 import edu.uci.ics.asterix.file.DataverseOperations;
@@ -98,7 +99,6 @@
 import edu.uci.ics.asterix.metadata.entities.InternalDatasetDetails;
 import edu.uci.ics.asterix.metadata.entities.NodeGroup;
 import edu.uci.ics.asterix.metadata.feeds.BuiltinFeedPolicies;
-import edu.uci.ics.asterix.metadata.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.metadata.feeds.FeedUtil;
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.ATypeTag;
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/ClusterWorkExecutor.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/ClusterWorkExecutor.java
index a6c1f8b..172a0ec 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/ClusterWorkExecutor.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/ClusterWorkExecutor.java
@@ -24,10 +24,8 @@
 import edu.uci.ics.asterix.event.schema.cluster.Node;
 import edu.uci.ics.asterix.metadata.api.IClusterManagementWork;
 import edu.uci.ics.asterix.metadata.cluster.AddNodeWork;
-import edu.uci.ics.asterix.metadata.cluster.AddNodeWorkResponse;
 import edu.uci.ics.asterix.metadata.cluster.ClusterManager;
 import edu.uci.ics.asterix.metadata.cluster.RemoveNodeWork;
-import edu.uci.ics.asterix.metadata.cluster.IClusterManagementWorkResponse.Status;
 import edu.uci.ics.asterix.om.util.AsterixClusterProperties;
 
 public class ClusterWorkExecutor implements Runnable {
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
index ecf1a14e..e0d7e6e 100755
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
@@ -23,6 +23,8 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.Unmarshaller;
@@ -30,18 +32,23 @@
 import edu.uci.ics.asterix.common.exceptions.ACIDException;
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
 import edu.uci.ics.asterix.common.functions.FunctionSignature;
-import edu.uci.ics.asterix.external.library.ExternalLibraryManager;
-import edu.uci.ics.asterix.external.library.Function;
-import edu.uci.ics.asterix.external.library.Functions;
-import edu.uci.ics.asterix.external.library.Library;
+import edu.uci.ics.asterix.external.library.ExternalLibrary;
+import edu.uci.ics.asterix.external.library.LibraryAdapter;
+import edu.uci.ics.asterix.external.library.LibraryFunction;
 import edu.uci.ics.asterix.metadata.MetadataManager;
 import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
 import edu.uci.ics.asterix.metadata.api.IMetadataEntity;
+import edu.uci.ics.asterix.metadata.entities.DatasourceAdapter;
+import edu.uci.ics.asterix.metadata.entities.DatasourceAdapter.AdapterType;
 import edu.uci.ics.asterix.metadata.entities.Dataverse;
+import edu.uci.ics.asterix.metadata.feeds.AdapterIdentifier;
+import edu.uci.ics.asterix.metadata.functions.ExternalLibraryManager;
 import edu.uci.ics.asterix.runtime.formats.NonTaggedDataFormat;
 
 public class ExternalLibraryBootstrap {
 
+    private static Logger LOGGER = Logger.getLogger(ExternalLibraryBootstrap.class.getName());
+
     public static void setUpExternaLibraries(boolean isMetadataNode) throws Exception {
 
         Map<String, List<String>> uninstalledLibs = null;
@@ -108,12 +115,21 @@
             List<edu.uci.ics.asterix.metadata.entities.Function> functions = MetadataManager.INSTANCE
                     .getDataverseFunctions(mdTxnCtx, dataverse);
             for (edu.uci.ics.asterix.metadata.entities.Function function : functions) {
-                if (function.getName().startsWith(libraryName + ":")) {
+                if (function.getName().startsWith(libraryName + "#")) {
                     MetadataManager.INSTANCE.dropFunction(mdTxnCtx, new FunctionSignature(dataverse,
                             function.getName(), function.getArity()));
                 }
             }
 
+            List<edu.uci.ics.asterix.metadata.entities.DatasourceAdapter> adapters = MetadataManager.INSTANCE
+                    .getDataverseAdapters(mdTxnCtx, dataverse);
+            for (edu.uci.ics.asterix.metadata.entities.DatasourceAdapter adapter : adapters) {
+                if (adapter.getAdapterIdentifier().getAdapterName().startsWith(libraryName + "#")) {
+                    MetadataManager.INSTANCE.dropAdapter(mdTxnCtx, dataverse, adapter.getAdapterIdentifier()
+                            .getAdapterName());
+                }
+            }
+
             MetadataManager.INSTANCE.dropLibrary(mdTxnCtx, dataverse, libraryName);
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
         } catch (Exception e) {
@@ -149,7 +165,7 @@
                 }
             });
 
-            Library library = getLibrary(new File(libraryDir + File.separator + libraryDescriptors[0]));
+            ExternalLibrary library = getLibrary(new File(libraryDir + File.separator + libraryDescriptors[0]));
 
             if (libraryDescriptors.length == 0) {
                 throw new Exception("No library descriptors defined");
@@ -162,7 +178,7 @@
                 MetadataManager.INSTANCE.addDataverse(mdTxnCtx, new Dataverse(dataverse,
                         NonTaggedDataFormat.NON_TAGGED_DATA_FORMAT, IMetadataEntity.PENDING_NO_OP));
             }
-            for (Function function : library.getFunctions().getFunction()) {
+            for (LibraryFunction function : library.getLibraryFunctions().getLibraryFunction()) {
                 String[] fargs = function.getArguments().trim().split(",");
                 List<String> args = new ArrayList<String>();
                 for (String arg : fargs) {
@@ -174,6 +190,14 @@
                 MetadataManager.INSTANCE.addFunction(mdTxnCtx, f);
             }
 
+            for (LibraryAdapter adapter : library.getLibraryAdapters().getLibraryAdapter()) {
+                String adapterFactoryClass = adapter.getFactoryClass();
+                String adapterName = libraryName + "#" + adapter.getName();
+                AdapterIdentifier aid = new AdapterIdentifier(dataverse, adapterName);
+                DatasourceAdapter dsa = new DatasourceAdapter(aid, adapterFactoryClass, AdapterType.EXTERNAL);
+                MetadataManager.INSTANCE.addAdapter(mdTxnCtx, dsa);
+            }
+
             MetadataManager.INSTANCE.addLibrary(mdTxnCtx, new edu.uci.ics.asterix.metadata.entities.Library(dataverse,
                     libraryName));
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -189,17 +213,20 @@
         ExternalLibraryManager.registerLibraryClassLoader(dataverse, libraryName, classLoader);
     }
 
-    private static Library getLibrary(File libraryXMLPath) throws Exception {
-        JAXBContext configCtx = JAXBContext.newInstance(Library.class);
+    private static ExternalLibrary getLibrary(File libraryXMLPath) throws Exception {
+        JAXBContext configCtx = JAXBContext.newInstance(ExternalLibrary.class);
         Unmarshaller unmarshaller = configCtx.createUnmarshaller();
-        Library library = (Library) unmarshaller.unmarshal(libraryXMLPath);
+        ExternalLibrary library = (ExternalLibrary) unmarshaller.unmarshal(libraryXMLPath);
         return library;
     }
 
     private static ClassLoader getLibraryClassLoader(String dataverse, String libraryName) throws Exception {
-        System.out.println(" installing lirbary " + libraryName + " in dataverse " + dataverse);
+
         File installDir = getLibraryInstallDir();
-        System.out.println(" install directory " + installDir.getAbsolutePath());
+        if (LOGGER.isLoggable(Level.INFO)) {
+            LOGGER.info("Installing lirbary " + libraryName + " in dataverse " + dataverse + "."
+                    + " Install Directory: " + installDir.getAbsolutePath());
+        }
 
         File libDir = new File(installDir.getAbsolutePath() + File.separator + dataverse + File.separator + libraryName);
         FilenameFilter jarFileFilter = new FilenameFilter() {
@@ -209,8 +236,6 @@
         };
 
         String[] jarsInLibDir = libDir.list(jarFileFilter);
-        System.out.println(" jars in lib dir " + jarsInLibDir);
-
         if (jarsInLibDir.length > 1) {
             throw new Exception("Incorrect library structure: found multiple library jars");
         }
@@ -238,6 +263,15 @@
                 urls[count++] = file.toURL();
             }
         }
+
+        if (LOGGER.isLoggable(Level.INFO)) {
+            StringBuilder logMesg = new StringBuilder("Classpath for library " + libraryName + "\n");
+            for (URL url : urls) {
+                logMesg.append(url.getFile() + "\n");
+            }
+            LOGGER.info(logMesg.toString());
+        }
+
         ClassLoader classLoader = new URLClassLoader(urls, parentClassLoader);
         return classLoader;
     }
@@ -253,48 +287,3 @@
     }
 
 }
-
-class ExternalLibrary {
-
-    private final String dataverse;
-    private final String name;
-    private final String language;
-    private final Functions functions;
-
-    public ExternalLibrary(String dataverse, String name, String language, Functions functions) {
-        this.dataverse = dataverse;
-        this.name = name;
-        this.language = language;
-        this.functions = functions;
-    }
-
-    public String toString() {
-        StringBuilder builder = new StringBuilder("");
-        builder.append("Library");
-        builder.append("\n");
-        builder.append("Functions");
-        builder.append("\n");
-        for (Function function : functions.getFunction()) {
-            builder.append(function);
-            builder.append("\n");
-        }
-        return new String(builder);
-    }
-
-    public String getDataverse() {
-        return dataverse;
-    }
-
-    public String getName() {
-        return name;
-    }
-
-    public String getLanguage() {
-        return language;
-    }
-
-    public Functions getFunctions() {
-        return functions;
-    }
-
-}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/FeedLifecycleListener.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/FeedLifecycleListener.java
index 37c6e8a..f0e51b2 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/FeedLifecycleListener.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/FeedLifecycleListener.java
@@ -41,6 +41,8 @@
 import edu.uci.ics.asterix.aql.expression.Identifier;
 import edu.uci.ics.asterix.aql.translator.AqlTranslator;
 import edu.uci.ics.asterix.common.exceptions.ACIDException;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
+import edu.uci.ics.asterix.common.feeds.SuperFeedManager;
 import edu.uci.ics.asterix.event.schema.cluster.Cluster;
 import edu.uci.ics.asterix.event.schema.cluster.Node;
 import edu.uci.ics.asterix.file.JobSpecificationUtils;
@@ -61,7 +63,6 @@
 import edu.uci.ics.asterix.metadata.entities.FeedActivity.FeedActivityType;
 import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
 import edu.uci.ics.asterix.metadata.feeds.BuiltinFeedPolicies;
-import edu.uci.ics.asterix.metadata.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.metadata.feeds.FeedIntakeOperatorDescriptor;
 import edu.uci.ics.asterix.metadata.feeds.FeedManagerElectMessage;
 import edu.uci.ics.asterix.metadata.feeds.FeedMetaOperatorDescriptor;
@@ -69,7 +70,6 @@
 import edu.uci.ics.asterix.metadata.feeds.IFeedMessage;
 import edu.uci.ics.asterix.metadata.feeds.MessageListener;
 import edu.uci.ics.asterix.metadata.feeds.MessageListener.IMessageAnalyzer;
-import edu.uci.ics.asterix.metadata.feeds.SuperFeedManager;
 import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
 import edu.uci.ics.asterix.om.util.AsterixClusterProperties;
 import edu.uci.ics.asterix.om.util.AsterixClusterProperties.State;
@@ -98,6 +98,11 @@
 import edu.uci.ics.hyracks.dataflow.std.misc.NullSinkOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexInsertUpdateDeleteOperatorDescriptor;
 
+/**
+ * A listener that subscribes to events associated with cluster membership (nodes joining/leaving the cluster)
+ * and job lifecycle (start/end of a job). Subscription to such events allows keeping track of feed ingestion jobs
+ * and take any corrective action that may be required when a node involved in a feed leaves the cluster.
+ */
 public class FeedLifecycleListener implements IJobLifecycleListener, IClusterEventsSubscriber, Serializable {
 
     private static final long serialVersionUID = 1L;
@@ -495,7 +500,7 @@
                 if (LOGGER.isLoggable(Level.WARNING)) {
                     LOGGER.warning("Revived Feed");
                 }
-             
+
             }
         }
 
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/FeedWorkRequestResponseHandler.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/FeedWorkRequestResponseHandler.java
index 3197bbb..fa27e7e 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/FeedWorkRequestResponseHandler.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/FeedWorkRequestResponseHandler.java
@@ -16,7 +16,6 @@
 
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -117,8 +116,6 @@
             failedNodeIds.add(feedFailure.nodeId);
         }
         List<String> chosenReplacements = new ArrayList<String>();
-        String metadataNodeName = AsterixAppContextInfo.getInstance().getMetadataProperties().getMetadataNodeName();
-        chosenReplacements.add(metadataNodeName);
         switch (resp.getStatus()) {
             case FAILURE:
                 for (FeedFailure feedFailure : feedFailures) {
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index 187394a..1f8ab08 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -196,12 +196,12 @@
     private void updateOnNodeJoin() {
         AsterixMetadataProperties metadataProperties = ((IAsterixPropertiesProvider) runtimeContext)
                 .getMetadataProperties();
-        AsterixTransactionProperties txnProperties = ((IAsterixPropertiesProvider) runtimeContext)
-                .getTransactionProperties();
         if (!metadataProperties.getNodeNames().contains(nodeId)) {
             metadataProperties.getNodeNames().add(nodeId);
             Cluster cluster = AsterixClusterProperties.INSTANCE.getCluster();
             String asterixInstanceName = cluster.getInstanceName();
+            AsterixTransactionProperties txnProperties = ((IAsterixPropertiesProvider) runtimeContext)
+                    .getTransactionProperties();
             Node self = null;
             for (Node node : cluster.getSubstituteNodes().getNode()) {
                 String ncId = asterixInstanceName + "_" + node.getId();
@@ -231,7 +231,6 @@
                 throw new IllegalStateException("Unknown node joining the cluster");
             }
         }
-
     }
 
     /**
diff --git a/asterix-app/src/test/resources/metadata/results/basic/meta15/meta15.1.adm b/asterix-app/src/test/resources/metadata/results/basic/meta15/meta15.1.adm
index 43cf9ec..47bf635 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/meta15/meta15.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/meta15/meta15.1.adm
@@ -6,5 +6,4 @@
 { "DataverseName": "Metadata", "Name": "localfs", "Classname": "edu.uci.ics.asterix.external.adapter.factory.NCFileSystemAdapterFactory", "Type": "INTERNAL", "Timestamp": "Tue Jul 16 22:38:45 PDT 2013" }
 { "DataverseName": "Metadata", "Name": "pull_twitter", "Classname": "edu.uci.ics.asterix.external.adapter.factory.PullBasedTwitterAdapterFactory", "Type": "INTERNAL", "Timestamp": "Tue Jul 16 22:38:45 PDT 2013" }
 { "DataverseName": "Metadata", "Name": "rss_feed", "Classname": "edu.uci.ics.asterix.external.adapter.factory.RSSFeedAdapterFactory", "Type": "INTERNAL", "Timestamp": "Tue Jul 16 22:38:45 PDT 2013" }
-{ "DataverseName": "Metadata", "Name": "synthetic_twitter_feed", "Classname": "edu.uci.ics.asterix.tools.external.data.SyntheticTwitterFeedAdapterFactory", "Type": "INTERNAL", "Timestamp": "Tue Jul 16 22:38:45 PDT 2013" }
 { "DataverseName": "Metadata", "Name": "twitter_firehose", "Classname": "edu.uci.ics.asterix.tools.external.data.TwitterFirehoseFeedAdapterFactory", "Type": "INTERNAL", "Timestamp": "Tue Jul 16 22:38:45 PDT 2013" }
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_03/feeds_03.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_03/feeds_03.1.ddl.aql
index 17bfcee..f15c481 100644
--- a/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_03/feeds_03.1.ddl.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_03/feeds_03.1.ddl.aql
@@ -26,5 +26,4 @@
 create feed TweetFeed
 using file_feed
 (("output-type-name"="TweetType"),("fs"="localfs"),("path"="nc1://data/twitter/obamatweets.adm"),("format"="adm"),("tuple-interval"="10"))
-apply function feed_processor; 
-
+apply function feed_processor;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_05/feeds_05.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_05/feeds_05.1.ddl.aql
index ec92972..1d4f619 100644
--- a/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_05/feeds_05.1.ddl.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_05/feeds_05.1.ddl.aql
@@ -1,6 +1,7 @@
 /*
  * Description  : Create a feed dataset that uses the synthetic feed simulator adapter. 
-                  The synthetic feed simulator uses the Social-Data generator to generate data and simulate a feed. The                   feed lasts a configured duration with data arriving at a configured rate (tweets per second). 
+                  The synthetic feed simulator uses the Social-Data generator to generate data and simulate a feed. 
+                  The feed lasts a configured duration with data arriving at a configured rate (tweets per second). 
                   Verify the existence of data after the feed finishes.
                   
  * Expected Res : Success
@@ -21,19 +22,17 @@
 } 
 
 create type TweetMessageType as closed {
-	tweetid: string,
-        user: TwitterUserType,
-        sender-location: point,
+	tweetid: int64,
+    user: TwitterUserType,
+    sender-location: point,
 	send-time: datetime,
-        referred-topics: {{ string }},
+    referred-topics: {{ string }},
 	message-text: string
 }
 
 create dataset SyntheticTweets(TweetMessageType)
 primary key tweetid;
 
-
 create feed  SyntheticTweetFeed
-using synthetic_twitter_feed
-(("duration"="5"),("tps"="2"),("dataverse-dataset"="feeds:SyntheticTweets"))
-
+using twitter_firehose
+(("duration"="5"),("tps"="50"),("tput-duration"="5"),("dataverse-dataset"="feeds:SyntheticTweets"),("mode"="controlled"));
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/IAsterixAppRuntimeContext.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/IAsterixAppRuntimeContext.java
index f05efe6..26ed19b 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/IAsterixAppRuntimeContext.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/IAsterixAppRuntimeContext.java
@@ -19,6 +19,7 @@
 
 import edu.uci.ics.asterix.common.exceptions.ACIDException;
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.common.feeds.IFeedManager;
 import edu.uci.ics.asterix.common.transactions.ITransactionSubsystem;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
 import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManager;
@@ -62,6 +63,8 @@
     public void deinitialize() throws HyracksDataException;
 
     public double getBloomFilterFalsePositiveRate();
-
+    
     public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID);
+
+    public IFeedManager getFeedManager();
 }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedConnectionId.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/FeedConnectionId.java
similarity index 97%
rename from asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedConnectionId.java
rename to asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/FeedConnectionId.java
index a487168..def7c10 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedConnectionId.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/FeedConnectionId.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.metadata.feeds;
+package edu.uci.ics.asterix.common.feeds;
 
 import java.io.Serializable;
 
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageService.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/FeedMessageService.java
similarity index 82%
rename from asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageService.java
rename to asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/FeedMessageService.java
index 40b2f26..3d0d8f9 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageService.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/FeedMessageService.java
@@ -1,4 +1,4 @@
-package edu.uci.ics.asterix.metadata.feeds;
+package edu.uci.ics.asterix.common.feeds;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -11,25 +11,29 @@
 
 /**
  * Sends feed report messages on behalf of an operator instance
- * to the SuperFeedMaanger associated with the feed.
+ * to the SuperFeedManager associated with the feed.
  */
 public class FeedMessageService {
 
     private static final Logger LOGGER = Logger.getLogger(FeedMessageService.class.getName());
+
+    public static final char MessageSeparator = '|';
     private static final char EOL = (char) "\n".getBytes()[0];
 
     private final FeedConnectionId feedId;
     private final LinkedBlockingQueue<String> inbox;
     private final FeedMessageHandler mesgHandler;
+    private final IFeedManager feedManager;
 
-    public FeedMessageService(FeedConnectionId feedId) {
+    public FeedMessageService(FeedConnectionId feedId, IFeedManager feedManager) {
         this.feedId = feedId;
-        this.inbox = new LinkedBlockingQueue<String>();
-        mesgHandler = new FeedMessageHandler(inbox, feedId);
+        inbox = new LinkedBlockingQueue<String>();
+        mesgHandler = new FeedMessageHandler(inbox, feedId, feedManager);
+        this.feedManager = feedManager;
     }
 
     public void start() throws UnknownHostException, IOException, Exception {
-        FeedManager.INSTANCE.getFeedExecutorService(feedId).execute(mesgHandler);
+        feedManager.getFeedExecutorService(feedId).execute(mesgHandler);
     }
 
     public void stop() throws IOException {
@@ -46,10 +50,12 @@
         private final FeedConnectionId feedId;
         private Socket sfmSocket;
         private boolean process = true;
+        private final IFeedManager feedManager;
 
-        public FeedMessageHandler(LinkedBlockingQueue<String> inbox, FeedConnectionId feedId) {
+        public FeedMessageHandler(LinkedBlockingQueue<String> inbox, FeedConnectionId feedId, IFeedManager feedManager) {
             this.inbox = inbox;
             this.feedId = feedId;
+            this.feedManager = feedManager;
         }
 
         public void run() {
@@ -66,7 +72,7 @@
                     }
                 }
                 if (LOGGER.isLoggable(Level.INFO)) {
-                    LOGGER.info("ENDED FEED MESSAGE SERVICE for " + feedId);
+                    LOGGER.info("Ended feed message service for " + feedId);
                 }
             } catch (Exception e) {
                 if (LOGGER.isLoggable(Level.WARNING)) {
@@ -95,11 +101,11 @@
 
         private Socket obtainSFMSocket() throws UnknownHostException, IOException, Exception {
             Socket sfmDirServiceSocket = null;
-            SuperFeedManager sfm = FeedManager.INSTANCE.getSuperFeedManager(feedId);
+            SuperFeedManager sfm = feedManager.getSuperFeedManager(feedId);
             try {
-                FeedRuntimeManager runtimeManager = FeedManager.INSTANCE.getFeedRuntimeManager(feedId);
+                FeedRuntimeManager runtimeManager = feedManager.getFeedRuntimeManager(feedId);
                 sfmDirServiceSocket = runtimeManager.createClientSocket(sfm.getHost(), sfm.getPort(),
-                        FeedManager.SOCKET_CONNECT_TIMEOUT);
+                        IFeedManager.SOCKET_CONNECT_TIMEOUT);
                 if (sfmDirServiceSocket == null) {
                     if (LOGGER.isLoggable(Level.WARNING)) {
                         LOGGER.warning("Unable to connect to " + sfm.getHost() + "[" + sfm.getPort() + "]");
@@ -126,7 +132,7 @@
                                 + sfm.getHost() + " " + port);
                     }
                     sfmSocket = runtimeManager.createClientSocket(sfm.getHost(), port,
-                            FeedManager.SOCKET_CONNECT_TIMEOUT);
+                            IFeedManager.SOCKET_CONNECT_TIMEOUT);
                 }
             } catch (Exception e) {
                 e.printStackTrace();
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/FeedReport.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/FeedReport.java
new file mode 100644
index 0000000..cda56ae
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/FeedReport.java
@@ -0,0 +1,116 @@
+package edu.uci.ics.asterix.common.feeds;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import edu.uci.ics.asterix.common.feeds.FeedRuntime.FeedRuntimeType;
+import edu.uci.ics.asterix.common.feeds.SuperFeedManager.FeedReportMessageType;
+
+public class FeedReport implements Comparable {
+
+    private FeedConnectionId feedId;
+    private FeedReportMessageType reportType;
+    private int partition = -1;
+    private FeedRuntimeType runtimeType;
+    private long value = -1;
+    private String[] representation;
+
+    public FeedReport() {
+    }
+
+    public FeedReport(String message) {
+        representation = message.split("\\|");
+    }
+
+    public void reset(String message) {
+        representation = message.split("\\|");
+        reportType = null;
+        feedId = null;
+        runtimeType = null;
+        partition = -1;
+        value = -1;
+    }
+
+    @Override
+    public String toString() {
+        return getFeedId() + " " + getReportType() + " " + getPartition() + " " + getRuntimeType() + " " + getValue();
+    }
+
+    public FeedConnectionId getFeedId() {
+        if (feedId == null) {
+            String feedIdRep = representation[1];
+            String[] feedIdComp = feedIdRep.split(":");
+            feedId = new FeedConnectionId(feedIdComp[0], feedIdComp[1], feedIdComp[2]);
+        }
+        return feedId;
+    }
+
+    public FeedReportMessageType getReportType() {
+        if (reportType == null) {
+            reportType = FeedReportMessageType.valueOf(representation[0].toUpperCase());
+        }
+        return reportType;
+    }
+
+    public int getPartition() {
+        if (partition < 0) {
+            partition = Integer.parseInt(representation[3]);
+        }
+        return partition;
+    }
+
+    public FeedRuntimeType getRuntimeType() {
+        if (runtimeType == null) {
+            runtimeType = FeedRuntimeType.valueOf(representation[2].toUpperCase());
+        }
+        return runtimeType;
+    }
+
+    public long getValue() {
+        if (value < 0) {
+            value = Long.parseLong(representation[4]);
+        }
+        return value;
+    }
+
+    public String[] getRepresentation() {
+        return representation;
+    }
+
+    @Override
+    public int compareTo(Object o) {
+        if (!(o instanceof FeedReport)) {
+            throw new IllegalArgumentException("Incorrect operand type " + o);
+        }
+
+        FeedReport other = (FeedReport) o;
+        if (!other.getReportType().equals(getReportType())) {
+            throw new IllegalArgumentException("Incorrect operand type " + o);
+        }
+
+        int returnValue = 0;
+
+        switch (getReportType()) {
+            case CONGESTION:
+                returnValue = ranking.get(getRuntimeType()) - ranking.get(other.getRuntimeType());
+                break;
+
+            case THROUGHPUT:
+                returnValue = (int) (other.getValue() - getValue());
+                break;
+        }
+
+        return returnValue;
+    }
+
+    private static Map<FeedRuntimeType, Integer> ranking = populateRanking();
+
+    private static Map<FeedRuntimeType, Integer> populateRanking() {
+        Map<FeedRuntimeType, Integer> ranking = new HashMap<FeedRuntimeType, Integer>();
+        ranking.put(FeedRuntimeType.INGESTION, 1);
+        ranking.put(FeedRuntimeType.COMPUTE, 2);
+        ranking.put(FeedRuntimeType.STORAGE, 3);
+        ranking.put(FeedRuntimeType.COMMIT, 4);
+        return ranking;
+    }
+}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedRuntime.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/FeedRuntime.java
similarity index 98%
rename from asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedRuntime.java
rename to asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/FeedRuntime.java
index 29b494e..d1b2faf 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedRuntime.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/FeedRuntime.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.metadata.feeds;
+package edu.uci.ics.asterix.common.feeds;
 
 import java.nio.ByteBuffer;
 
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedRuntimeManager.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/FeedRuntimeManager.java
similarity index 84%
rename from asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedRuntimeManager.java
rename to asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/FeedRuntimeManager.java
index b4951ec..92aa4ee 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedRuntimeManager.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/FeedRuntimeManager.java
@@ -1,4 +1,18 @@
-package edu.uci.ics.asterix.metadata.feeds;
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.common.feeds;
 
 import java.io.IOException;
 import java.net.ServerSocket;
@@ -13,17 +27,17 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import edu.uci.ics.asterix.metadata.feeds.FeedRuntime.FeedRuntimeId;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime.FeedRuntimeId;
 
 public class FeedRuntimeManager {
 
     private static Logger LOGGER = Logger.getLogger(FeedRuntimeManager.class.getName());
 
     private final FeedConnectionId feedId;
+    private final IFeedManager feedManager;
     private SuperFeedManager superFeedManager;
     private final Map<FeedRuntimeId, FeedRuntime> feedRuntimes;
     private final ExecutorService executorService;
@@ -31,11 +45,12 @@
     private SocketFactory socketFactory = new SocketFactory();
     private final LinkedBlockingQueue<String> feedReportQueue;
 
-    public FeedRuntimeManager(FeedConnectionId feedId) {
+    public FeedRuntimeManager(FeedConnectionId feedId, IFeedManager feedManager) {
         this.feedId = feedId;
         feedRuntimes = new ConcurrentHashMap<FeedRuntimeId, FeedRuntime>();
         executorService = Executors.newCachedThreadPool();
         feedReportQueue = new LinkedBlockingQueue<String>();
+        this.feedManager = feedManager;
     }
 
     public void close(boolean closeAll) throws IOException {
@@ -44,7 +59,7 @@
         if (messageService != null) {
             messageService.stop();
             if (LOGGER.isLoggable(Level.INFO)) {
-                LOGGER.info("Shut down message service s for :" + feedId);
+                LOGGER.info("Shut down message services for :" + feedId);
             }
             messageService = null;
         }
@@ -73,7 +88,7 @@
         if (LOGGER.isLoggable(Level.INFO)) {
             LOGGER.info("Started Super Feed Manager for feed :" + feedId);
         }
-        this.messageService = new FeedMessageService(feedId);
+        this.messageService = new FeedMessageService(feedId, feedManager);
         messageService.start();
         if (LOGGER.isLoggable(Level.INFO)) {
             LOGGER.info("Started Feed Message Service for feed :" + feedId);
@@ -92,7 +107,7 @@
         feedRuntimes.put(runtimeId, feedRuntime);
     }
 
-    public void deregisterFeedRuntime(FeedRuntimeId runtimeId) {
+    public void deregisterFeedRuntime(FeedRuntimeId runtimeId)  {
         feedRuntimes.remove(runtimeId);
         if (feedRuntimes.isEmpty()) {
             synchronized (this) {
@@ -100,7 +115,7 @@
                     if (LOGGER.isLoggable(Level.INFO)) {
                         LOGGER.info("De-registering feed");
                     }
-                    FeedManager.INSTANCE.deregisterFeed(runtimeId.getFeedId());
+                    feedManager.deregisterFeed(runtimeId.getFeedId());
                 }
             }
         }
@@ -173,12 +188,6 @@
             return socket;
         }
 
-        public ServerSocket createServerSocket() throws IOException {
-            ServerSocket socket = new ServerSocket(0);
-            serverSockets.add(socket);
-            return socket;
-        }
-
         private static class SocketId {
             private final String host;
             private final int port;
@@ -188,11 +197,6 @@
                 this.port = port;
             }
 
-            public SocketId(int port) {
-                this.host = "127.0.0.1";
-                this.port = port;
-            }
-
             public String getHost() {
                 return host;
             }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IFeedManager.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/IFeedManager.java
similarity index 61%
rename from asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IFeedManager.java
rename to asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/IFeedManager.java
index ac33966..6cdc45c 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IFeedManager.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/IFeedManager.java
@@ -12,42 +12,54 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.metadata.feeds;
+package edu.uci.ics.asterix.common.feeds;
 
 import java.io.IOException;
 import java.util.concurrent.ExecutorService;
 
-import edu.uci.ics.asterix.metadata.feeds.FeedRuntime.FeedRuntimeId;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime.FeedRuntimeId;
 
 /**
  * Handle (de)registration of feeds for delivery of control messages.
  */
 public interface IFeedManager {
 
+    public static final long SOCKET_CONNECT_TIMEOUT = 5000;
+
     /**
+     * Returns the executor service associated with the feed.
+     * 
      * @param feedId
      * @return
      */
     public ExecutorService getFeedExecutorService(FeedConnectionId feedId);
 
     /**
+     * Allows registration of a feedRuntime.
+     * 
      * @param feedRuntime
      * @throws Exception
      */
     public void registerFeedRuntime(FeedRuntime feedRuntime) throws Exception;
 
     /**
+     * Allows de-registration of a feed runtime.
+     * 
      * @param feedRuntimeId
      */
     public void deRegisterFeedRuntime(FeedRuntimeId feedRuntimeId);
 
     /**
+     * Obtain feed runtime corresponding to a feedRuntimeId
+     * 
      * @param feedRuntimeId
      * @return
      */
     public FeedRuntime getFeedRuntime(FeedRuntimeId feedRuntimeId);
 
     /**
+     * Register the Super Feed Manager associated witht a feed.
+     * 
      * @param feedId
      * @param sfm
      * @throws Exception
@@ -55,15 +67,35 @@
     public void registerSuperFeedManager(FeedConnectionId feedId, SuperFeedManager sfm) throws Exception;
 
     /**
+     * Obtain a handle to the Super Feed Manager associated with the feed.
+     * 
      * @param feedId
      * @return
      */
     public SuperFeedManager getSuperFeedManager(FeedConnectionId feedId);
 
     /**
+     * De-register a feed
+     * 
      * @param feedId
      * @throws IOException
      */
-    void deregisterFeed(FeedConnectionId feedId) throws IOException;
+    void deregisterFeed(FeedConnectionId feedId);
+
+    /**
+     * Obtain the feed runtime manager associated with a feed.
+     * 
+     * @param feedId
+     * @return
+     */
+    public FeedRuntimeManager getFeedRuntimeManager(FeedConnectionId feedId);
+
+    /**
+     * Obtain a handle to the feed Message service associated with a feed.
+     * 
+     * @param feedId
+     * @return
+     */
+    public FeedMessageService getFeedMessageService(FeedConnectionId feedId);
 
 }
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/MessageListener.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/MessageListener.java
new file mode 100644
index 0000000..7beb212
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/MessageListener.java
@@ -0,0 +1,179 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.common.feeds;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.nio.CharBuffer;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class MessageListener {
+
+    private static final Logger LOGGER = Logger.getLogger(MessageListener.class.getName());
+
+    private int port;
+    private final LinkedBlockingQueue<String> outbox;
+
+    private ExecutorService executorService = Executors.newFixedThreadPool(10);
+
+    private MessageListenerServer listenerServer;
+
+    public MessageListener(int port, LinkedBlockingQueue<String> outbox) {
+        this.port = port;
+        this.outbox = outbox;
+    }
+
+    public void stop() {
+        listenerServer.stop();
+        System.out.println("STOPPED MESSAGE RECEIVING SERVICE AT " + port);
+        if (!executorService.isShutdown()) {
+            executorService.shutdownNow();
+        }
+
+    }
+
+    public void start() throws IOException {
+        System.out.println("STARTING MESSAGE RECEIVING SERVICE AT " + port);
+        listenerServer = new MessageListenerServer(port, outbox);
+        executorService.execute(listenerServer);
+    }
+
+    private static class MessageListenerServer implements Runnable {
+
+        private final int port;
+        private final LinkedBlockingQueue<String> outbox;
+        private ServerSocket server;
+
+        public MessageListenerServer(int port, LinkedBlockingQueue<String> outbox) {
+            this.port = port;
+            this.outbox = outbox;
+        }
+
+        public void stop() {
+            try {
+                server.close();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+
+        @Override
+        public void run() {
+            char EOL = (char) "\n".getBytes()[0];
+            Socket client = null;
+            try {
+                server = new ServerSocket(port);
+                client = server.accept();
+                InputStream in = client.getInputStream();
+                CharBuffer buffer = CharBuffer.allocate(5000);
+                char ch;
+                while (true) {
+                    ch = (char) in.read();
+                    if (((int) ch) == -1) {
+                        break;
+                    }
+                    while (ch != EOL) {
+                        buffer.put(ch);
+                        ch = (char) in.read();
+                    }
+                    buffer.flip();
+                    String s = new String(buffer.array());
+                    synchronized (outbox) {
+                        outbox.add(s + "\n");
+                    }
+                    buffer.position(0);
+                    buffer.limit(5000);
+                }
+
+            } catch (Exception e) {
+                if (LOGGER.isLoggable(Level.WARNING)) {
+                    LOGGER.warning("Unable to start Message listener" + server);
+                }
+            } finally {
+                if (server != null) {
+                    try {
+                        server.close();
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    }
+                }
+            }
+
+        }
+
+    }
+
+    private static class MessageParser implements Runnable {
+
+        private Socket client;
+        private IMessageAnalyzer messageAnalyzer;
+        private static final char EOL = (char) "\n".getBytes()[0];
+
+        public MessageParser(Socket client, IMessageAnalyzer messageAnalyzer) {
+            this.client = client;
+            this.messageAnalyzer = messageAnalyzer;
+        }
+
+        @Override
+        public void run() {
+            CharBuffer buffer = CharBuffer.allocate(5000);
+            char ch;
+            try {
+                InputStream in = client.getInputStream();
+                while (true) {
+                    ch = (char) in.read();
+                    if (((int) ch) == -1) {
+                        break;
+                    }
+                    while (ch != EOL) {
+                        buffer.put(ch);
+                        ch = (char) in.read();
+                    }
+                    buffer.flip();
+                    String s = new String(buffer.array());
+                    synchronized (messageAnalyzer) {
+                        messageAnalyzer.getMessageQueue().add(s + "\n");
+                    }
+                    buffer.position(0);
+                    buffer.limit(5000);
+                }
+            } catch (IOException ioe) {
+                ioe.printStackTrace();
+            } finally {
+                try {
+                    client.close();
+                } catch (IOException ioe) {
+                    // do nothing
+                }
+            }
+        }
+    }
+
+    public static interface IMessageAnalyzer {
+
+        /**
+         * @return
+         */
+        public LinkedBlockingQueue<String> getMessageQueue();
+
+    }
+
+}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/SuperFeedManager.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/SuperFeedManager.java
similarity index 91%
rename from asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/SuperFeedManager.java
rename to asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/SuperFeedManager.java
index a45456b6..37306a0 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/SuperFeedManager.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/feeds/SuperFeedManager.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.metadata.feeds;
+package edu.uci.ics.asterix.common.feeds;
 
 import java.io.IOException;
 import java.io.OutputStream;
@@ -29,10 +29,6 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import edu.uci.ics.asterix.metadata.feeds.FeedRuntime.FeedRuntimeType;
-import edu.uci.ics.asterix.metadata.feeds.MessageListener.IMessageAnalyzer;
-import edu.uci.ics.asterix.om.util.AsterixClusterProperties;
-
 /**
  * The feed operators running in an NC report their health (statistics) to the local Feed Manager.
  * A feed thus has a Feed Manager per NC. From amongst the Feed Maanger, a SuperFeedManager is chosen (randomly)
@@ -85,6 +81,8 @@
 
     private boolean started = false;
 
+    private final IFeedManager feedManager;
+
     public static final int PORT_RANGE_ASSIGNED = 10;
 
     public enum FeedReportMessageType {
@@ -92,8 +90,10 @@
         THROUGHPUT
     }
 
-    public SuperFeedManager(FeedConnectionId feedId, String host, String nodeId, int port) throws Exception {
+    public SuperFeedManager(FeedConnectionId feedId, String host, String nodeId, int port, IFeedManager feedManager)
+            throws Exception {
         this.feedConnectionId = feedId;
+        this.feedManager = feedManager;
         this.nodeId = nodeId;
         this.feedReportPort = port;
         this.feedReportSubscribePort = port + 1;
@@ -128,12 +128,12 @@
 
     public void start() throws IOException {
         if (sfmService == null) {
-            ExecutorService executorService = FeedManager.INSTANCE.getFeedExecutorService(feedConnectionId);
+            ExecutorService executorService = feedManager.getFeedExecutorService(feedConnectionId);
             sfmService = new FeedReportDestinationSocketProvider(feedReportPort, feedReportInbox, feedConnectionId,
-                    availablePort);
+                    availablePort, feedManager);
             executorService.execute(sfmService);
             subscriptionService = new SuperFeedReportSubscriptionService(feedConnectionId, feedReportSubscribePort,
-                    sfmService.getMesgAnalyzer(), availablePort);
+                    sfmService.getMesgAnalyzer(), availablePort, feedManager);
             executorService.execute(subscriptionService);
         }
         if (LOGGER.isLoggable(Level.INFO)) {
@@ -174,13 +174,15 @@
         private String EOM = "\n";
         private final FeedReportProvider reportProvider;
         private final List<FeedDataProviderService> dataProviders = new ArrayList<FeedDataProviderService>();
+        private final IFeedManager feedManager;
 
         public SuperFeedReportSubscriptionService(FeedConnectionId feedId, int port, FeedReportProvider reportProvider,
-                AtomicInteger nextPort) throws IOException {
+                AtomicInteger nextPort, IFeedManager feedManager) throws IOException {
             this.feedId = feedId;
-            serverFeedSubscribe = FeedManager.INSTANCE.getFeedRuntimeManager(feedId).createServerSocket(port);
+            serverFeedSubscribe = feedManager.getFeedRuntimeManager(feedId).createServerSocket(port);
             this.subscriptionPort = nextPort;
             this.reportProvider = reportProvider;
+            this.feedManager = feedManager;
         }
 
         public void stop() {
@@ -199,10 +201,10 @@
                     int port = subscriptionPort.incrementAndGet();
                     LinkedBlockingQueue<String> reportInbox = new LinkedBlockingQueue<String>();
                     reportProvider.registerSubsription(reportInbox);
-                    FeedDataProviderService dataProviderService = new FeedDataProviderService(feedId, port, reportInbox);
+                    FeedDataProviderService dataProviderService = new FeedDataProviderService(feedId, port,
+                            reportInbox, feedManager);
                     dataProviders.add(dataProviderService);
-                    FeedManager.INSTANCE.getFeedRuntimeManager(feedId).getExecutorService()
-                            .execute(dataProviderService);
+                    feedManager.getFeedRuntimeManager(feedId).getExecutorService().execute(dataProviderService);
                     if (LOGGER.isLoggable(Level.INFO)) {
                         LOGGER.info("Recevied subscription request for feed :" + feedId
                                 + " Subscripton available at port " + subscriptionPort);
@@ -219,18 +221,20 @@
     private static class FeedDataProviderService implements Runnable {
 
         private final FeedConnectionId feedId;
+        private final IFeedManager feedManager;
         private int subscriptionPort;
         private ServerSocket dataProviderSocket;
         private LinkedBlockingQueue<String> inbox;
         private boolean active = true;
         private String EOM = "\n";
 
-        public FeedDataProviderService(FeedConnectionId feedId, int port, LinkedBlockingQueue<String> inbox)
-                throws IOException {
+        public FeedDataProviderService(FeedConnectionId feedId, int port, LinkedBlockingQueue<String> inbox,
+                IFeedManager feedManager) throws IOException {
             this.feedId = feedId;
             this.subscriptionPort = port;
             this.inbox = inbox;
-            dataProviderSocket = FeedManager.INSTANCE.getFeedRuntimeManager(feedId).createServerSocket(port);
+            dataProviderSocket = feedManager.getFeedRuntimeManager(feedId).createServerSocket(port);
+            this.feedManager = feedManager;
         }
 
         @Override
@@ -279,15 +283,15 @@
         private boolean process = true;
 
         public FeedReportDestinationSocketProvider(int port, LinkedBlockingQueue<String> inbox,
-                FeedConnectionId feedId, AtomicInteger availablePort) throws IOException {
-            FeedRuntimeManager runtimeManager = FeedManager.INSTANCE.getFeedRuntimeManager(feedId);
+                FeedConnectionId feedId, AtomicInteger availablePort, IFeedManager feedManager) throws IOException {
+            FeedRuntimeManager runtimeManager = feedManager.getFeedRuntimeManager(feedId);
             this.feedReportSocket = runtimeManager.createServerSocket(port);
             this.nextPort = availablePort;
             this.inbox = inbox;
             this.feedId = feedId;
             this.messageListeners = new ArrayList<MessageListener>();
             this.mesgAnalyzer = new FeedReportProvider(inbox, feedId);
-            FeedManager.INSTANCE.getFeedExecutorService(feedId).execute(mesgAnalyzer);
+            feedManager.getFeedExecutorService(feedId).execute(mesgAnalyzer);
         }
 
         public void stop() {
@@ -407,7 +411,7 @@
                                     String tp = ingestionThroughputs.get(i + "");
                                     if (tp != null) {
                                         ingestionThroughputs.put(i + "", null);
-                                        finalMessage.append(tp + "|");
+                                        finalMessage.append(tp + FeedMessageService.MessageSeparator);
                                     } else {
                                         dispatchReport = false;
                                         break;
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/HDFSAdapterFactory.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/HDFSAdapterFactory.java
index f81c0d5..b10ebcf 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/HDFSAdapterFactory.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/HDFSAdapterFactory.java
@@ -25,7 +25,6 @@
 import edu.uci.ics.asterix.external.dataset.adapter.HDFSAdapter;
 import edu.uci.ics.asterix.metadata.feeds.IDatasourceAdapter;
 import edu.uci.ics.asterix.metadata.feeds.IGenericAdapterFactory;
-import edu.uci.ics.asterix.metadata.feeds.ITypedAdapterFactory;
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java
index 2e68588..1a8c0df 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java
@@ -116,12 +116,12 @@
 
     protected INodeResolver getNodeResolver() {
         if (nodeResolver == null) {
-            nodeResolver = initNodeResolver();
+            nodeResolver = initializeNodeResolver();
         }
         return nodeResolver;
     }
 
-    private static INodeResolver initNodeResolver() {
+    private static INodeResolver initializeNodeResolver() {
         INodeResolver nodeResolver = null;
         String configuredNodeResolverFactory = System.getProperty(NODE_RESOLVER_FACTORY_PROPERTY);
         if (configuredNodeResolverFactory != null) {
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/RSSFeedAdapterFactory.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
index c7f25b0..41f1d56 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
@@ -19,7 +19,6 @@
 import java.util.Map;
 
 import edu.uci.ics.asterix.external.dataset.adapter.RSSFeedAdapter;
-import edu.uci.ics.asterix.metadata.feeds.IAdapterFactory;
 import edu.uci.ics.asterix.metadata.feeds.IDatasourceAdapter;
 import edu.uci.ics.asterix.metadata.feeds.ITypedAdapterFactory;
 import edu.uci.ics.asterix.om.types.ARecordType;
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/StreamBasedAdapterFactory.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/StreamBasedAdapterFactory.java
index 8d578f7..f69eb2d 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/StreamBasedAdapterFactory.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/StreamBasedAdapterFactory.java
@@ -1,15 +1,26 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package edu.uci.ics.asterix.external.adapter.factory;
 
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
-import edu.uci.ics.asterix.external.util.DNSResolverFactory;
 import edu.uci.ics.asterix.external.util.INodeResolver;
-import edu.uci.ics.asterix.external.util.INodeResolverFactory;
 import edu.uci.ics.asterix.metadata.feeds.ConditionalPushTupleParserFactory;
 import edu.uci.ics.asterix.metadata.feeds.IAdapterFactory;
 import edu.uci.ics.asterix.om.types.ARecordType;
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/IPushBasedFeedClient.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/IPushBasedFeedClient.java
deleted file mode 100644
index da49240..0000000
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/IPushBasedFeedClient.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.external.dataset.adapter;
-
-import java.io.InputStream;
-import java.util.Map;
-
-import edu.uci.ics.asterix.common.exceptions.AsterixException;
-
-public interface IPushBasedFeedClient {
-
-    /**
-     * @return
-     * @throws AsterixException
-     */
-    public InputStream getInputStream() throws AsterixException;
-
-    /**
-     * Provides logic for any corrective action that feed client needs to execute on
-     * encountering an exception.
-     * 
-     * @param e
-     *            The exception encountered during fetching of data from external source
-     * @throws AsterixException
-     */
-    public void resetOnFailure(Exception e) throws AsterixException;
-
-    /**
-     * @param configuration
-     */
-    public boolean alter(Map<String, Object> configuration);
-
-    public void stop();
-
-}
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedAdapter.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedAdapter.java
index da2264d..e2a4b76 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedAdapter.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedAdapter.java
@@ -32,9 +32,9 @@
 import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
 
 /**
- * Acts as an abstract class for all pull-based external data adapters.
- * Captures the common logic for obtaining bytes from an external source
- * and packing them into frames as tuples.
+ * Acts as an abstract class for all pull-based external data adapters. Captures
+ * the common logic for obtaining bytes from an external source and packing them
+ * into frames as tuples.
  */
 public abstract class PullBasedAdapter extends AbstractFeedDatasourceAdapter implements IDatasourceAdapter,
         IFeedAdapter {
@@ -45,14 +45,13 @@
     protected ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(1);
     protected IPullBasedFeedClient pullBasedFeedClient;
     protected ARecordType adapterOutputType;
+    protected boolean continueIngestion = true;
+    protected Map<String, String> configuration;
+
     private FrameTupleAppender appender;
     private ByteBuffer frame;
-    protected boolean continueIngestion = true;
-    protected boolean alterRequested = false;
-    private Map<String, String> modifiedConfiguration = null;
     private long tupleCount = 0;
     private final IHyracksTaskContext ctx;
-    protected Map<String, String> configuration;
 
     public abstract IPullBasedFeedClient getFeedClient(int partition) throws Exception;
 
@@ -65,10 +64,6 @@
         return tupleCount;
     }
 
-    public void alter(Map<String, String> modifedConfiguration) {
-        this.modifiedConfiguration = modifedConfiguration;
-    }
-
     @Override
     public void start(int partition, IFrameWriter writer) throws Exception {
         appender = new FrameTupleAppender(ctx.getFrameSize());
@@ -97,17 +92,11 @@
                     case DATA_NOT_AVAILABLE:
                         break;
                 }
-                if (alterRequested) {
-                    boolean success = pullBasedFeedClient.alter(modifiedConfiguration);
-                    if (success) {
-                        configuration = modifiedConfiguration;
-                        modifiedConfiguration = null;
-                    }
-                }
+
             } catch (Exception failureException) {
                 try {
                     failureException.printStackTrace();
-                    boolean continueIngestion = policyEnforcer.handleSoftwareFailure(failureException);
+                    boolean continueIngestion = policyEnforcer.continueIngestionPostSoftwareFailure(failureException);
                     if (continueIngestion) {
                         pullBasedFeedClient.resetOnFailure(failureException);
                         tupleBuilder.reset();
@@ -124,9 +113,7 @@
 
     private void appendTupleToFrame(IFrameWriter writer) throws HyracksDataException {
         if (!appender.append(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray(), 0, tupleBuilder.getSize())) {
-            System.out.println("flushing frame");
             FrameUtils.flushFrame(frame, writer);
-            System.out.println("flushed frame");
             appender.reset(frame, true);
             if (!appender.append(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray(), 0,
                     tupleBuilder.getSize())) {
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedTwitterAdapter.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedTwitterAdapter.java
index 41143d8..019d1b7 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedTwitterAdapter.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/PullBasedTwitterAdapter.java
@@ -19,8 +19,6 @@
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
 import edu.uci.ics.asterix.metadata.feeds.IFeedAdapter;
 import edu.uci.ics.asterix.om.types.ARecordType;
-import edu.uci.ics.asterix.om.types.BuiltinType;
-import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
 
 /**
@@ -54,19 +52,6 @@
         tweetClient.stop();
     }
 
-    @Override
-    public void alter(Map<String, String> properties) {
-        alterRequested = true;
-    }
-
-    public boolean isAlterRequested() {
-        return alterRequested;
-    }
-
-    public void postAlteration() {
-        alterRequested = false;
-    }
-
     public ARecordType getAdapterOutputType() {
         return recordType;
     }
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/RSSFeedAdapter.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/RSSFeedAdapter.java
index 102482d..6ace5c5 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/RSSFeedAdapter.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/dataset/adapter/RSSFeedAdapter.java
@@ -15,7 +15,6 @@
 package edu.uci.ics.asterix.external.dataset.adapter;
 
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -33,8 +32,6 @@
 
     private List<String> feedURLs = new ArrayList<String>();
     private boolean isStopRequested = false;
-    private boolean isAlterRequested = false;
-    private Map<String, String> alteredParams = new HashMap<String, String>();
     private String id_prefix = "";
 
     private IPullBasedFeedClient rssFeedClient;
@@ -56,13 +53,6 @@
     }
 
     @Override
-    public void alter(Map<String, String> properties) {
-        isAlterRequested = true;
-        this.alteredParams = properties;
-        reconfigure(properties);
-    }
-
-    @Override
     public void stop() {
         isStopRequested = true;
     }
@@ -82,14 +72,6 @@
         }
     }
 
-    public boolean isAlterRequested() {
-        return isAlterRequested;
-    }
-
-    public Map<String, String> getAlteredParams() {
-        return alteredParams;
-    }
-
     @Override
     public IPullBasedFeedClient getFeedClient(int partition) throws Exception {
         if (rssFeedClient == null) {
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/ExternalFunction.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/ExternalFunction.java
index c922a5e..c4c40fd 100755
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/ExternalFunction.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/ExternalFunction.java
@@ -18,6 +18,7 @@
 
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
 import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import edu.uci.ics.asterix.metadata.functions.ExternalLibraryManager;
 import edu.uci.ics.asterix.om.functions.IExternalFunctionInfo;
 import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/ExternalFunctionDescriptorProvider.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/ExternalFunctionDescriptorProvider.java
index b5888a6..b3914e2 100755
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/ExternalFunctionDescriptorProvider.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/ExternalFunctionDescriptorProvider.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package edu.uci.ics.asterix.external.library;
 
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/ExternalScalarFunctionEvaluatorFactory.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/ExternalScalarFunctionEvaluatorFactory.java
index a185000..4e3f9fc 100755
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/ExternalScalarFunctionEvaluatorFactory.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/ExternalScalarFunctionEvaluatorFactory.java
@@ -22,6 +22,7 @@
 
 public class ExternalScalarFunctionEvaluatorFactory implements ICopyEvaluatorFactory {
 
+    private static final long serialVersionUID = 1L;
     private final IExternalFunctionInfo finfo;
     private final ICopyEvaluatorFactory[] args;
 
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/JavaFunctionHelper.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/JavaFunctionHelper.java
index 8aed123..192cf3e 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/JavaFunctionHelper.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/JavaFunctionHelper.java
@@ -49,7 +49,6 @@
     private final IDataOutputProvider outputProvider;
     private IJObject[] arguments;
     private IJObject resultHolder;
-    private IAObject innerResult;
     private ISerializerDeserializer resultSerde;
     private IObjectPool<IJObject, IAType> objectPool = new ListObjectPool<IJObject, IAType>(new JTypeObjectFactory());
     byte[] buffer = new byte[32768];
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/RuntimeExternalFunctionUtil.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/RuntimeExternalFunctionUtil.java
index a5cac66..c0b6945 100755
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/RuntimeExternalFunctionUtil.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/RuntimeExternalFunctionUtil.java
@@ -14,7 +14,6 @@
  */
 package edu.uci.ics.asterix.external.library;
 
-import java.io.File;
 import java.nio.ByteBuffer;
 import java.util.HashMap;
 import java.util.Map;
@@ -98,15 +97,5 @@
         }
     }
 
-    public static File getExternalLibraryDeployDir(String nodeId) {
-        String filePath = null;
-        if (nodeId != null) {
-            filePath = "edu.uci.ics.hyracks.control.nc.NodeControllerService" + "/" + nodeId + "/"
-                    + "applications/asterix/expanded/external-lib/libraries";
-        } else {
-            filePath = "ClusterControllerService" + "/" + "applications/asterix/expanded/external-lib/libraries";
-
-        }
-        return new File(filePath);
-    }
+    
 }
diff --git a/asterix-external-data/src/main/resources/feed_policy/basic.pol b/asterix-external-data/src/main/resources/feed_policy/basic.pol
deleted file mode 100644
index 8aed9f8..0000000
--- a/asterix-external-data/src/main/resources/feed_policy/basic.pol
+++ /dev/null
@@ -1,8 +0,0 @@
-feed.policy.name=Basic
-
-software.failure.persist.exception=false
-software.failure.continue.on.failure=false
-hardware.failure.auto.restart=false
-
-statistics.collect=false
-elastic=false
diff --git a/asterix-external-data/src/main/resources/feed_policy/basic_monitored.pol b/asterix-external-data/src/main/resources/feed_policy/basic_monitored.pol
deleted file mode 100644
index efca65e..0000000
--- a/asterix-external-data/src/main/resources/feed_policy/basic_monitored.pol
+++ /dev/null
@@ -1,11 +0,0 @@
-feed.policy.name=Basic_Monitored
-
-software.failure.persist.exception=true
-software.failure.continue.on.failure=false
-hardware.failure.auto.restart=false
-
-statistics.collect=true
-statistics.collect.period=60
-statistics.collect.period.unit=sec
-
-elastic=false
diff --git a/asterix-external-data/src/main/resources/feed_policy/mission_critical.pol b/asterix-external-data/src/main/resources/feed_policy/mission_critical.pol
deleted file mode 100644
index 0a1da08..0000000
--- a/asterix-external-data/src/main/resources/feed_policy/mission_critical.pol
+++ /dev/null
@@ -1,17 +0,0 @@
-feed.policy.name=Mission_Critical
-
-software.failure.persist.exception=true
-software.failure.continue.on.failure=true
-hardware.failure.auto.restart=true
-
-statistics.collect=true
-statistics.collect.period=60
-statistics.collect.period.unit=sec
-
-elastic=true
-criterion.ingestion.rate.percent=true
-criterion.ingestion.rate.percent.delta=50
-
-criterion.ingestion.rate.absolute=true
-criterion.ingestion.rate.absolute.delta=2000
-criterion.ingestion.rate.absolute.unit=sec
diff --git a/asterix-external-data/src/main/resources/feed_policy/twitter.pol b/asterix-external-data/src/main/resources/feed_policy/twitter.pol
deleted file mode 100644
index 5fa507f..0000000
--- a/asterix-external-data/src/main/resources/feed_policy/twitter.pol
+++ /dev/null
@@ -1,17 +0,0 @@
-feed.policy.name=Twitter
-
-software.failure.persist.exception=true
-software.failure.continue.on.failure=true
-hardware.failure.auto.restart=false
-
-statistics.collect=true
-statistics.collect.period=60
-statistics.collect.period.unit=sec
-
-elastic=true
-criterion.ingestion.rate.percent=true
-criterion.ingestion.rate.percent.delta=50
-
-criterion.ingestion.rate.absolute=true
-criterion.ingestion.rate.absolute.delta=2000
-criterion.ingestion.rate.absolute.unit=sec
diff --git a/asterix-external-data/src/main/resources/schema/library.xsd b/asterix-external-data/src/main/resources/schema/library.xsd
index 3dc4659..f58175a 100644
--- a/asterix-external-data/src/main/resources/schema/library.xsd
+++ b/asterix-external-data/src/main/resources/schema/library.xsd
@@ -1,42 +1,66 @@
 <?xml version="1.0" encoding="ISO-8859-1" ?>
-<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:lib="library" targetNamespace="library" elementFormDefault="qualified">
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
+	xmlns:lib="library" targetNamespace="library" elementFormDefault="qualified">
 
-<!-- definition of simple types --> 
-<xs:element name="language" type="xs:string"/>
-<xs:element name="name" type="xs:string"/>
-<xs:element name="arguments" type="xs:string"/>
-<xs:element name="return_type" type="xs:string"/>
-<xs:element name="function_type" type="xs:string"/>
-<xs:element name="definition" type="xs:string"/>
+	<!-- definition of simple types -->
+	<xs:element name="language" type="xs:string" />
+	<xs:element name="name" type="xs:string" />
+	<xs:element name="arguments" type="xs:string" />
+	<xs:element name="return_type" type="xs:string" />
+	<xs:element name="function_type" type="xs:string" />
+	<xs:element name="definition" type="xs:string" />
 
-<!-- definition of complex elements -->
-<xs:element name="function">
-  <xs:complexType>
-    <xs:sequence>
-      <xs:element ref="lib:name"/>
-      <xs:element ref="lib:function_type"/>
-      <xs:element ref="lib:arguments"/>
-      <xs:element ref="lib:return_type"/>
-      <xs:element ref="lib:definition"/>
-    </xs:sequence>
-  </xs:complexType>
-</xs:element>
+	<xs:element name="factory_class" type="xs:string" />
+	<xs:element name="adaptor_type" type="xs:string" />
 
-<xs:element name="functions">
-  <xs:complexType>
-    <xs:sequence>
-      <xs:element ref="lib:function" maxOccurs="unbounded"/>
-    </xs:sequence>
-  </xs:complexType>
-</xs:element>
 
-<xs:element name="library">
-  <xs:complexType>
-    <xs:sequence>
-      <xs:element ref="lib:language"/>
-      <xs:element ref="lib:functions" minOccurs="0"/>
-    </xs:sequence>
-  </xs:complexType>
-</xs:element>
+	<!-- definition of complex elements -->
+	<xs:element name="libraryFunction">
+		<xs:complexType>
+			<xs:sequence>
+				<xs:element ref="lib:name" />
+				<xs:element ref="lib:function_type" />
+				<xs:element ref="lib:arguments" />
+				<xs:element ref="lib:return_type" />
+				<xs:element ref="lib:definition" />
+			</xs:sequence>
+		</xs:complexType>
+	</xs:element>
+
+	<xs:element name="libraryFunctions">
+		<xs:complexType>
+			<xs:sequence>
+				<xs:element ref="lib:libraryFunction" maxOccurs="unbounded" />
+			</xs:sequence>
+		</xs:complexType>
+	</xs:element>
+
+	<xs:element name="libraryAdapter">
+		<xs:complexType>
+			<xs:sequence>
+				<xs:element ref="lib:name" />
+				<xs:element ref="lib:factory_class" />
+				<xs:element ref="lib:adaptor_type" />
+			</xs:sequence>
+		</xs:complexType>
+	</xs:element>
+
+	<xs:element name="libraryAdapters">
+		<xs:complexType>
+			<xs:sequence>
+				<xs:element ref="lib:libraryAdapter" maxOccurs="unbounded" />
+			</xs:sequence>
+		</xs:complexType>
+	</xs:element>
+
+	<xs:element name="externalLibrary">
+		<xs:complexType>
+			<xs:sequence>
+				<xs:element ref="lib:language" />
+				<xs:element ref="lib:libraryFunctions" minOccurs="0" />
+			    <xs:element ref="lib:libraryAdapters" minOccurs="0" />
+			</xs:sequence>
+		</xs:complexType>
+	</xs:element>
 
 </xs:schema>     
diff --git a/asterix-external-data/src/test/java/edu/uci/ics/asterix/external/library/CapitalFinderFunction.java b/asterix-external-data/src/test/java/edu/uci/ics/asterix/external/library/CapitalFinderFunction.java
index 7953b62..8a11757 100644
--- a/asterix-external-data/src/test/java/edu/uci/ics/asterix/external/library/CapitalFinderFunction.java
+++ b/asterix-external-data/src/test/java/edu/uci/ics/asterix/external/library/CapitalFinderFunction.java
@@ -29,16 +29,14 @@
 
     @Override
     public void deinitialize() {
-        System.out.println(" De Initialized");
+        System.out.println("De-Initialized");
     }
 
     @Override
     public void evaluate(IFunctionHelper functionHelper) throws Exception {
         JString country = ((JString) functionHelper.getArgument(0));
         JRecord record = (JRecord) functionHelper.getResultObject();
-        //     ((JString) record.getValueByName("country")).setValue(country.getValue());
         String capitalCity = capitalList.getProperty(country.getValue(), NOT_FOUND);
-        //      ((JString) record.getValueByName("capital")).setValue(capitalCity);
         capital.setValue(capitalCity);
 
         record.setField("country", country);
diff --git a/asterix-external-data/src/test/java/edu/uci/ics/asterix/external/library/EchoDelayFunction.java b/asterix-external-data/src/test/java/edu/uci/ics/asterix/external/library/EchoDelayFunction.java
index 5f9be77..ce63eec 100644
--- a/asterix-external-data/src/test/java/edu/uci/ics/asterix/external/library/EchoDelayFunction.java
+++ b/asterix-external-data/src/test/java/edu/uci/ics/asterix/external/library/EchoDelayFunction.java
@@ -40,7 +40,7 @@
     public void evaluate(IFunctionHelper functionHelper) throws Exception {
         JRecord inputRecord = (JRecord) functionHelper.getArgument(0);
         long sleepInterval = rand.nextInt(range);
-     //   Thread.sleep(5);
+        Thread.sleep(sleepInterval);
         functionHelper.setResult(inputRecord);
     }
 }
diff --git a/asterix-external-data/src/test/java/edu/uci/ics/asterix/external/library/SumFunction.java b/asterix-external-data/src/test/java/edu/uci/ics/asterix/external/library/SumFunction.java
index 0a4eca6..96eba97 100644
--- a/asterix-external-data/src/test/java/edu/uci/ics/asterix/external/library/SumFunction.java
+++ b/asterix-external-data/src/test/java/edu/uci/ics/asterix/external/library/SumFunction.java
@@ -24,7 +24,7 @@
 
     @Override
     public void deinitialize() {
-        System.out.println(" De Initialized");
+        // nothing to do here
     }
 
     @Override
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataCache.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataCache.java
index f78eb31..c36726b 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataCache.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataCache.java
@@ -20,6 +20,7 @@
 import java.util.List;
 import java.util.Map;
 
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.common.functions.FunctionSignature;
 import edu.uci.ics.asterix.metadata.api.IMetadataEntity;
 import edu.uci.ics.asterix.metadata.entities.CompactionPolicy;
@@ -34,7 +35,6 @@
 import edu.uci.ics.asterix.metadata.entities.Index;
 import edu.uci.ics.asterix.metadata.entities.Library;
 import edu.uci.ics.asterix.metadata.entities.NodeGroup;
-import edu.uci.ics.asterix.metadata.feeds.FeedConnectionId;
 
 /**
  * Caches metadata entities such that the MetadataManager does not have to
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
index 590890a..99bd06a 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
@@ -22,6 +22,7 @@
 
 import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
 import edu.uci.ics.asterix.common.exceptions.ACIDException;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.common.functions.FunctionSignature;
 import edu.uci.ics.asterix.common.transactions.JobId;
 import edu.uci.ics.asterix.metadata.api.IAsterixStateProxy;
@@ -41,7 +42,6 @@
 import edu.uci.ics.asterix.metadata.entities.Library;
 import edu.uci.ics.asterix.metadata.entities.Node;
 import edu.uci.ics.asterix.metadata.entities.NodeGroup;
-import edu.uci.ics.asterix.metadata.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.transaction.management.service.transaction.JobIdFactory;
 import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 
@@ -790,4 +790,15 @@
         return feedActivities;
     }
 
+    public List<DatasourceAdapter> getDataverseAdapters(MetadataTransactionContext mdTxnCtx, String dataverse)
+            throws MetadataException {
+        List<DatasourceAdapter> dataverseAdapters;
+        try {
+            dataverseAdapters = metadataNode.getDataverseAdapters(mdTxnCtx.getJobId(), dataverse);
+        } catch (RemoteException e) {
+            throw new MetadataException(e);
+        }
+        return dataverseAdapters;
+    }
+
 }
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
index 3c7e4a3..a962837 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
@@ -29,6 +29,7 @@
 import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
 import edu.uci.ics.asterix.common.exceptions.ACIDException;
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.common.functions.FunctionSignature;
 import edu.uci.ics.asterix.common.transactions.AbstractOperationCallback;
 import edu.uci.ics.asterix.common.transactions.DatasetId;
@@ -71,7 +72,6 @@
 import edu.uci.ics.asterix.metadata.entitytupletranslators.NodeGroupTupleTranslator;
 import edu.uci.ics.asterix.metadata.entitytupletranslators.NodeTupleTranslator;
 import edu.uci.ics.asterix.metadata.feeds.FeedActivityIdFactory;
-import edu.uci.ics.asterix.metadata.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.metadata.valueextractors.DatasetNameValueExtractor;
 import edu.uci.ics.asterix.metadata.valueextractors.DatatypeNameValueExtractor;
 import edu.uci.ics.asterix.metadata.valueextractors.MetadataEntityValueExtractor;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataTransactionContext.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataTransactionContext.java
index 1c990af..6859e45 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataTransactionContext.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataTransactionContext.java
@@ -16,7 +16,6 @@
 package edu.uci.ics.asterix.metadata;
 
 import java.util.ArrayList;
-import java.util.Map;
 
 import edu.uci.ics.asterix.common.functions.FunctionSignature;
 import edu.uci.ics.asterix.common.transactions.JobId;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataManager.java
index f23275c..23612c7 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataManager.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataManager.java
@@ -19,6 +19,7 @@
 import java.util.List;
 
 import edu.uci.ics.asterix.common.exceptions.ACIDException;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.common.functions.FunctionSignature;
 import edu.uci.ics.asterix.metadata.MetadataException;
 import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
@@ -36,7 +37,6 @@
 import edu.uci.ics.asterix.metadata.entities.Library;
 import edu.uci.ics.asterix.metadata.entities.Node;
 import edu.uci.ics.asterix.metadata.entities.NodeGroup;
-import edu.uci.ics.asterix.metadata.feeds.FeedConnectionId;
 
 /**
  * A metadata manager provides user access to Asterix metadata (e.g., types,
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataNode.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataNode.java
index 8434847..ddfd622 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataNode.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IMetadataNode.java
@@ -21,6 +21,7 @@
 import java.util.List;
 
 import edu.uci.ics.asterix.common.exceptions.ACIDException;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.common.functions.FunctionSignature;
 import edu.uci.ics.asterix.common.transactions.JobId;
 import edu.uci.ics.asterix.metadata.MetadataException;
@@ -38,7 +39,6 @@
 import edu.uci.ics.asterix.metadata.entities.Library;
 import edu.uci.ics.asterix.metadata.entities.Node;
 import edu.uci.ics.asterix.metadata.entities.NodeGroup;
-import edu.uci.ics.asterix.metadata.feeds.FeedConnectionId;
 
 /**
  * A metadata node stores metadata in its local storage structures (currently
@@ -449,6 +449,14 @@
     public List<DatasourceAdapter> getDataverseAdapters(JobId jobId, String dataverseName) throws MetadataException,
             RemoteException;
 
+    /**
+     * @param jobId
+     * @param dataverseName
+     * @param adapterName
+     * @return
+     * @throws MetadataException
+     * @throws RemoteException
+     */
     public DatasourceAdapter getAdapter(JobId jobId, String dataverseName, String adapterName)
             throws MetadataException, RemoteException;
 
@@ -499,7 +507,6 @@
     public void initializeFeedActivityIdFactory(JobId jobId) throws MetadataException, RemoteException;
 
     /**
-     * 
      * @param jobId
      * @param dataverse
      * @param policy
@@ -511,7 +518,6 @@
             RemoteException;
 
     /**
-     * 
      * @param jobId
      * @throws MetadataException
      * @throws RemoteException
@@ -519,7 +525,6 @@
     public void initializeDatasetIdFactory(JobId jobId) throws MetadataException, RemoteException;
 
     /**
-     * 
      * @return
      * @throws MetadataException
      * @throws RemoteException
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
index 0b2a6c7..f2c8b5c 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -319,8 +319,7 @@
                 "edu.uci.ics.asterix.external.adapter.factory.CNNFeedAdapterFactory",
                 "edu.uci.ics.asterix.tools.external.data.RateControlledFileSystemBasedAdapterFactory",
                 "edu.uci.ics.asterix.tools.external.data.TwitterFirehoseFeedAdapterFactory",
-                "edu.uci.ics.asterix.tools.external.data.GenericSocketFeedAdapterFactory",
-                "edu.uci.ics.asterix.tools.external.data.SyntheticTwitterFeedAdapterFactory" };
+                "edu.uci.ics.asterix.tools.external.data.GenericSocketFeedAdapterFactory" };
         DatasourceAdapter adapter;
         for (String adapterClassName : builtInAdapterClassNames) {
             adapter = getAdapter(adapterClassName);
@@ -333,7 +332,7 @@
             MetadataManager.INSTANCE.addFeedPolicy(mdTxnCtx, feedPolicy);
         }
     }
-  
+
     private static void insertInitialCompactionPolicies(MetadataTransactionContext mdTxnCtx) throws Exception {
         String[] builtInCompactionPolicyClassNames = new String[] {
                 "edu.uci.ics.hyracks.storage.am.lsm.common.impls.ConstantMergePolicyFactory",
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/dataset/hints/DatasetHints.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/dataset/hints/DatasetHints.java
index 2a80963..e27a4cb 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/dataset/hints/DatasetHints.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/dataset/hints/DatasetHints.java
@@ -17,7 +17,6 @@
 import java.util.HashSet;
 import java.util.Set;
 
-import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
 import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
 import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
 
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
index 46b8268..5b24d38 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
@@ -33,6 +33,7 @@
 import edu.uci.ics.asterix.common.dataflow.AsterixLSMInvertedIndexInsertDeleteOperatorDescriptor;
 import edu.uci.ics.asterix.common.dataflow.AsterixLSMTreeInsertDeleteOperatorDescriptor;
 import edu.uci.ics.asterix.common.dataflow.IAsterixApplicationContextInfo;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
 import edu.uci.ics.asterix.common.ioopcallbacks.LSMInvertedIndexIOOperationCallbackFactory;
 import edu.uci.ics.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallbackFactory;
@@ -63,7 +64,6 @@
 import edu.uci.ics.asterix.metadata.feeds.BuiltinFeedPolicies;
 import edu.uci.ics.asterix.metadata.feeds.EndFeedMessage;
 import edu.uci.ics.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
-import edu.uci.ics.asterix.metadata.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.metadata.feeds.FeedIntakeOperatorDescriptor;
 import edu.uci.ics.asterix.metadata.feeds.FeedMessageOperatorDescriptor;
 import edu.uci.ics.asterix.metadata.feeds.FeedUtil;
@@ -115,7 +115,6 @@
 import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
 import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
 import edu.uci.ics.hyracks.algebricks.runtime.operators.std.SinkWriterRuntimeFactory;
-import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
 import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
 import edu.uci.ics.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
@@ -165,7 +164,6 @@
     private boolean asyncResults;
     private ResultSetId resultSetId;
     private IResultSerializerFactoryProvider resultSerializerFactoryProvider;
-    private IHyracksClientConnection hcc;
 
     private final Dataverse defaultDataverse;
     private JobId jobId;
@@ -364,13 +362,12 @@
         } catch (AlgebricksException ae) {
             throw ae;
         } catch (Exception e) {
-            e.printStackTrace();
             throw new AlgebricksException("Unable to create adapter " + e);
         }
 
         if (!(adapterFactory.getSupportedOperations().equals(SupportedOperation.READ) || adapterFactory
                 .getSupportedOperations().equals(SupportedOperation.READ_WRITE))) {
-            throw new AlgebricksException("external dataset adapter does not support read operation");
+            throw new AlgebricksException(" External dataset adapter does not support read operation");
         }
 
         ISerializerDeserializer payloadSerde = format.getSerdeProvider().getSerializerDeserializer(itemType);
@@ -431,7 +428,9 @@
 
             FeedPolicy feedPolicy = (FeedPolicy) ((AqlDataSource) dataSource).getProperties().get(
                     BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY);
-
+            if (feedPolicy == null) {
+                throw new AlgebricksException("Feed not configured with a policy");
+            }
             feedPolicy.getProperties().put(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY, feedPolicy.getPolicyName());
             feedIngestor = new FeedIntakeOperatorDescriptor(jobSpec, new FeedConnectionId(
                     feedDataSource.getDatasourceDataverse(), feedDataSource.getDatasourceName(), feedDataSource
@@ -460,21 +459,9 @@
         List<String> feedLocations = new ArrayList<String>();
         String[] ingestLocs = feedActivity.getFeedActivityDetails().get(FeedActivityDetails.INGEST_LOCATIONS)
                 .split(",");
-        String[] computeLocs = feedActivity.getFeedActivityDetails().get(FeedActivityDetails.COMPUTE_LOCATIONS)
-                .split(",");
-        String[] storageLocs = feedActivity.getFeedActivityDetails().get(FeedActivityDetails.STORAGE_LOCATIONS)
-                .split(",");
-
         for (String loc : ingestLocs) {
             feedLocations.add(loc);
         }
-        for (String loc : computeLocs) {
-            feedLocations.add(loc);
-        }
-        for (String loc : storageLocs) {
-            feedLocations.add(loc);
-        }
-
         FeedConnectionId feedId = new FeedConnectionId(dataverse, feedName, dataset);
         String[] locations = feedLocations.toArray(new String[] {});
         IFeedMessage feedMessage = new EndFeedMessage(feedId);
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/FeedDataSource.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/FeedDataSource.java
index fc5d9ca..f729683 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/FeedDataSource.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/FeedDataSource.java
@@ -14,10 +14,10 @@
  */
 package edu.uci.ics.asterix.metadata.declared;
 
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.metadata.MetadataManager;
 import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
 import edu.uci.ics.asterix.metadata.entities.Feed;
-import edu.uci.ics.asterix.metadata.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
@@ -34,6 +34,7 @@
         feed = null;
         MetadataTransactionContext ctx = null;
         try {
+            MetadataManager.INSTANCE.acquireReadLatch();
             ctx = MetadataManager.INSTANCE.beginTransaction();
             feed = MetadataManager.INSTANCE.getFeed(ctx, feedId.getDataverse(), feedId.getFeedName());
             if (feed == null) {
@@ -51,6 +52,8 @@
                 }
             }
 
+        } finally {
+            MetadataManager.INSTANCE.releaseReadLatch();
         }
     }
 
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/Feed.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/Feed.java
index 5d1704e..6a63f28 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/Feed.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/Feed.java
@@ -17,9 +17,7 @@
 
 import java.util.Map;
 
-import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
 import edu.uci.ics.asterix.common.functions.FunctionSignature;
-import edu.uci.ics.asterix.metadata.IDatasetDetails;
 import edu.uci.ics.asterix.metadata.MetadataCache;
 import edu.uci.ics.asterix.metadata.api.IMetadataEntity;
 
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/AdapterRuntimeManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/AdapterRuntimeManager.java
index 44ad1e9..1380d9e 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/AdapterRuntimeManager.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/AdapterRuntimeManager.java
@@ -19,8 +19,10 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime.FeedRuntimeType;
+import edu.uci.ics.asterix.common.feeds.IFeedManager;
 import edu.uci.ics.asterix.metadata.feeds.FeedFrameWriter.Mode;
-import edu.uci.ics.asterix.metadata.feeds.FeedRuntime.FeedRuntimeType;
 import edu.uci.ics.hyracks.api.comm.IFrameWriter;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
 
@@ -40,6 +42,8 @@
 
     private IngestionRuntime ingestionRuntime;
 
+    private final IFeedManager feedManager;
+
     public enum State {
         /*
          * Indicates that data from external source will be pushed downstream for storage 
@@ -56,10 +60,11 @@
     }
 
     public AdapterRuntimeManager(FeedConnectionId feedId, IFeedAdapter feedAdapter, FeedFrameWriter writer,
-            int partition, LinkedBlockingQueue<IFeedMessage> inbox) {
+            int partition, LinkedBlockingQueue<IFeedMessage> inbox, IFeedManager feedManager) {
         this.feedId = feedId;
         this.feedAdapter = feedAdapter;
         this.partition = partition;
+        this.feedManager = feedManager;
         this.adapterExecutor = new AdapterExecutor(partition, writer, feedAdapter, this);
     }
 
@@ -67,11 +72,11 @@
     public void start() throws Exception {
         state = State.ACTIVE_INGESTION;
         ingestionRuntime = new IngestionRuntime(feedId, partition, FeedRuntimeType.INGESTION, this);
-        FeedManager.INSTANCE.registerFeedRuntime(ingestionRuntime);
+        feedManager.registerFeedRuntime(ingestionRuntime);
         if (LOGGER.isLoggable(Level.INFO)) {
             LOGGER.info("Registered feed runtime manager for " + this.getFeedId());
         }
-        ExecutorService executorService = FeedManager.INSTANCE.getFeedExecutorService(feedId);
+        ExecutorService executorService = feedManager.getFeedExecutorService(feedId);
         executorService.execute(adapterExecutor);
     }
 
@@ -83,9 +88,9 @@
             synchronized (this) {
                 notifyAll();
             }
-        } catch (Exception e) {
-            if (LOGGER.isLoggable(Level.WARNING)) {
-                LOGGER.warning("Unable to stop adapter");
+        } catch (Exception exception) {
+            if (LOGGER.isLoggable(Level.SEVERE)) {
+                LOGGER.severe("Unable to stop adapter " + feedAdapter + ", encountered exception " + exception);
             }
         }
     }
@@ -136,7 +141,6 @@
                 if (LOGGER.isLoggable(Level.SEVERE)) {
                     LOGGER.severe("Exception during feed ingestion " + e.getMessage());
                 }
-                e.printStackTrace();
             } finally {
                 synchronized (runtimeManager) {
                     runtimeManager.notifyAll();
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/BuiltinFeedPolicies.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/BuiltinFeedPolicies.java
index 3db5916..da5fdde 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/BuiltinFeedPolicies.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/BuiltinFeedPolicies.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package edu.uci.ics.asterix.metadata.feeds;
 
 import java.util.HashMap;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/ConditionalPushTupleParserFactory.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/ConditionalPushTupleParserFactory.java
index 2dfe886..62893e2 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/ConditionalPushTupleParserFactory.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/ConditionalPushTupleParserFactory.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package edu.uci.ics.asterix.metadata.feeds;
 
 import java.io.IOException;
@@ -6,7 +20,6 @@
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.logging.Level;
-import java.util.logging.Logger;
 
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
 import edu.uci.ics.asterix.om.types.ARecordType;
@@ -26,8 +39,6 @@
 
     private static final long serialVersionUID = 1L;
 
-    private static final Logger LOGGER = Logger.getLogger(ConditionalPushTupleParserFactory.class.getName());
-
     @Override
     public ITupleParser createTupleParser(IHyracksTaskContext ctx) throws HyracksDataException {
         IDataParser dataParser = null;
@@ -47,7 +58,6 @@
     private IValueParserFactory[] valueParserFactories;
     private char delimiter;
     private final ParserType parserType;
-    private Object lock;
 
     public enum ParserType {
         ADM,
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/EndFeedMessage.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/EndFeedMessage.java
index fffee64..2d56658 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/EndFeedMessage.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/EndFeedMessage.java
@@ -14,6 +14,8 @@
  */
 package edu.uci.ics.asterix.metadata.feeds;
 
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
+
 /**
  * A feed control message indicating the need to end the feed. This message is dispatched
  * to all locations that host an operator invovled in the feed pipeline.
@@ -29,12 +31,8 @@
         this.feedId = feedId;
     }
 
-    public FeedConnectionId getFeedId() {
-        return feedId;
-    }
-
     @Override
     public String toString() {
-        return "" + feedId;
+        return MessageType.END.name() + feedId;
     }
 }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedActivityIdFactory.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedActivityIdFactory.java
index db1a62a..2109425 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedActivityIdFactory.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedActivityIdFactory.java
@@ -1,25 +1,36 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package edu.uci.ics.asterix.metadata.feeds;
 
 import java.util.concurrent.atomic.AtomicInteger;
 
 public class FeedActivityIdFactory {
-	private static AtomicInteger id = new AtomicInteger();
-	private static boolean isInitialized = false;
+    private static AtomicInteger id = new AtomicInteger();
+    private static boolean isInitialized = false;
 
-	public static boolean isInitialized() {
-		return isInitialized;
-	}
+    public static boolean isInitialized() {
+        return isInitialized;
+    }
 
-	public static void initialize(int initialId) {
-		id.set(initialId);
-		isInitialized = true;
-	}
+    public static void initialize(int initialId) {
+        id.set(initialId);
+        isInitialized = true;
+    }
 
-	public static int generateFeedActivityId() {
-		return id.incrementAndGet();
-	}
+    public static int generateFeedActivityId() {
+        return id.incrementAndGet();
+    }
 
-	public static int getMostRecentFeedActivityId() {
-		return id.get();
-	}
 }
\ No newline at end of file
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedFrameWriter.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedFrameWriter.java
index 53038d2..9e76c21 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedFrameWriter.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedFrameWriter.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package edu.uci.ics.asterix.metadata.feeds;
 
 import java.io.IOException;
@@ -10,8 +24,12 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import edu.uci.ics.asterix.metadata.feeds.FeedRuntime.FeedRuntimeType;
-import edu.uci.ics.asterix.metadata.feeds.SuperFeedManager.FeedReportMessageType;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
+import edu.uci.ics.asterix.common.feeds.FeedMessageService;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime.FeedRuntimeType;
+import edu.uci.ics.asterix.common.feeds.IFeedManager;
+import edu.uci.ics.asterix.common.feeds.SuperFeedManager;
+import edu.uci.ics.asterix.common.feeds.SuperFeedManager.FeedReportMessageType;
 import edu.uci.ics.hyracks.api.comm.IFrameWriter;
 import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
@@ -29,7 +47,7 @@
     private static final Logger LOGGER = Logger.getLogger(FeedFrameWriter.class.getName());
 
     /** The threshold for the time required in pushing a frame to the network. **/
-    public static final long FLUSH_THRESHOLD_TIME = 5000; // 3 seconds
+    public static final long FLUSH_THRESHOLD_TIME = 5000; // 5 seconds
 
     /** Actual frame writer provided to an operator. **/
     private IFrameWriter writer;
@@ -53,18 +71,18 @@
 
     /**
      * Detects if the operator is unable to push a frame downstream
-     * within a threshold period of time. In addition, measure the
+     * within a threshold period of time. In addition, it measure the
      * throughput as observed on the output channel of the associated operator.
      */
     private HealthMonitor healthMonitor;
 
     /**
-     * Manager scheduling of tasks
+     * A Timer instance for managing scheduling of tasks.
      */
     private Timer timer;
 
     /**
-     * Provides access to the tuples in a frame. Used in collecting statistics.
+     * Provides access to the tuples in a frame. Used in collecting statistics
      */
     private FrameTupleAccessor fta;
 
@@ -79,22 +97,22 @@
         /**
          * Failure mode of operation for an operator when
          * input frames are not pushed to the downstream operator but
-         * are buffered for future retrieval.
+         * are buffered for future retrieval. This mode is adopted
+         * during failure recovery.
          */
         STORE
     }
 
     public FeedFrameWriter(IFrameWriter writer, IOperatorNodePushable nodePushable, FeedConnectionId feedId,
             FeedPolicyEnforcer policyEnforcer, String nodeId, FeedRuntimeType feedRuntimeType, int partition,
-            FrameTupleAccessor fta) {
+            FrameTupleAccessor fta, IFeedManager feedManager) {
         this.writer = writer;
         this.mode = Mode.FORWARD;
         this.nodePushable = nodePushable;
         this.reportHealth = policyEnforcer.getFeedPolicyAccessor().collectStatistics();
         if (reportHealth) {
             timer = new Timer();
-            healthMonitor = new HealthMonitor(FLUSH_THRESHOLD_TIME, feedId, nodeId, feedRuntimeType, partition,
-                    FLUSH_THRESHOLD_TIME, timer);
+            healthMonitor = new HealthMonitor(feedId, nodeId, feedRuntimeType, partition, timer, fta, feedManager);
             if (LOGGER.isLoggable(Level.INFO)) {
                 LOGGER.info("Statistics collection enabled for the feed " + feedId + " " + feedRuntimeType + " ["
                         + partition + "]");
@@ -130,9 +148,9 @@
                 try {
                     if (reportHealth) {
                         fta.reset(buffer);
-                        healthMonitor.notifyStart();
+                        healthMonitor.notifyStartFrameFlushActivity();
                         writer.nextFrame(buffer);
-                        healthMonitor.notifyFinish(fta.getTupleCount());
+                        healthMonitor.notifyFinishFrameFlushActivity();
                     } else {
                         writer.nextFrame(buffer);
                     }
@@ -143,55 +161,71 @@
                 }
                 if (frames.size() > 0) {
                     for (ByteBuffer buf : frames) {
-                        if (LOGGER.isLoggable(Level.WARNING)) {
-                            LOGGER.warning("Flusing OLD frame: " + buf + " on behalf of "
-                                    + nodePushable.getDisplayName());
-                        }
                         writer.nextFrame(buf);
+                        if (LOGGER.isLoggable(Level.WARNING)) {
+                            LOGGER.warning("Flushed old frame (from previous failed execution) : " + buf
+                                    + " on behalf of " + nodePushable.getDisplayName());
+                        }
                     }
+                    frames.clear();
                 }
-                frames.clear();
                 break;
             case STORE:
+
+                /* TODO:
+                 * Limit the in-memory space utilized during the STORE mode. The limit (expressed in bytes) 
+                 * is a parameter specified as part of the feed ingestion policy. Below is a basic implemenation
+                 * that allocates a buffer on demand.   
+                 * */
+
                 ByteBuffer storageBuffer = ByteBuffer.allocate(buffer.capacity());
                 storageBuffer.put(buffer);
                 frames.add(storageBuffer);
                 storageBuffer.flip();
                 if (LOGGER.isLoggable(Level.INFO)) {
-                    LOGGER.info("Stored frame for " + nodePushable);
+                    LOGGER.info("Stored frame for " + nodePushable.getDisplayName());
                 }
                 break;
         }
     }
 
+    /**
+     * Detects if the operator is unable to push a frame downstream
+     * within a threshold period of time. In addition, it measure the
+     * throughput as observed on the output channel of the associated operator.
+     */
     private static class HealthMonitor extends TimerTask {
 
+        private static final String EOL = "\n";
+
         private long startTime = -1;
         private FramePushState state;
-        private long flushThresholdTime;
-        private static final String EOL = "\n";
-        private FeedConnectionId feedId;
-        private String nodeId;
-        private FeedRuntimeType feedRuntimeType;
-        private int partition;
         private AtomicLong numTuplesInInterval = new AtomicLong(0);
-        private long period;
         private boolean collectThroughput;
         private FeedMessageService mesgService;
 
-        public HealthMonitor(long flushThresholdTime, FeedConnectionId feedId, String nodeId,
-                FeedRuntimeType feedRuntimeType, int partition, long period, Timer timer) {
-            this.flushThresholdTime = flushThresholdTime;
+        private final FeedConnectionId feedId;
+        private final String nodeId;
+        private final FeedRuntimeType feedRuntimeType;
+        private final int partition;
+        private final long period;
+        private final FrameTupleAccessor fta;
+        private final IFeedManager feedManager;
+
+        public HealthMonitor(FeedConnectionId feedId, String nodeId, FeedRuntimeType feedRuntimeType, int partition,
+                Timer timer, FrameTupleAccessor fta, IFeedManager feedManager) {
             this.state = FramePushState.INTIALIZED;
             this.feedId = feedId;
             this.nodeId = nodeId;
             this.feedRuntimeType = feedRuntimeType;
             this.partition = partition;
-            this.period = period;
+            this.period = FLUSH_THRESHOLD_TIME;
             this.collectThroughput = feedRuntimeType.equals(FeedRuntimeType.INGESTION);
+            this.fta = fta;
+            this.feedManager = feedManager;
         }
 
-        public void notifyStart() {
+        public void notifyStartFrameFlushActivity() {
             startTime = System.currentTimeMillis();
             state = FramePushState.WAITING_FOR_FLUSH_COMPLETION;
         }
@@ -202,55 +236,57 @@
          */
         public void reset() {
             mesgService = null;
-            collectThroughput = true;
+            collectThroughput = feedRuntimeType.equals(FeedRuntimeType.INGESTION);
         }
 
-        public void notifyFinish(int numTuples) {
+        public void notifyFinishFrameFlushActivity() {
             state = FramePushState.WAITNG_FOR_NEXT_FRAME;
-            numTuplesInInterval.set(numTuplesInInterval.get() + numTuples);
+            numTuplesInInterval.set(numTuplesInInterval.get() + fta.getTupleCount());
         }
 
         @Override
         public void run() {
             if (state.equals(FramePushState.WAITING_FOR_FLUSH_COMPLETION)) {
                 long currentTime = System.currentTimeMillis();
-                if (currentTime - startTime > flushThresholdTime) {
+                if (currentTime - startTime > FLUSH_THRESHOLD_TIME) {
                     if (LOGGER.isLoggable(Level.SEVERE)) {
                         LOGGER.severe("Congestion reported by " + feedRuntimeType + " [" + partition + "]");
                     }
-                    sendReportToSFM(currentTime - startTime, FeedReportMessageType.CONGESTION,
+                    sendReportToSuperFeedManager(currentTime - startTime, FeedReportMessageType.CONGESTION,
                             System.currentTimeMillis());
                 }
             }
             if (collectThroughput) {
                 int instantTput = (int) Math.ceil((((double) numTuplesInInterval.get() * 1000) / period));
-                System.out.println("MEASURED TPUT:" + numTuplesInInterval.get());
-                sendReportToSFM(instantTput, FeedReportMessageType.THROUGHPUT, System.currentTimeMillis());
+                sendReportToSuperFeedManager(instantTput, FeedReportMessageType.THROUGHPUT, System.currentTimeMillis());
             }
             numTuplesInInterval.set(0);
         }
 
-        private void sendReportToSFM(long value, SuperFeedManager.FeedReportMessageType mesgType, long timestamp) {
+        private void sendReportToSuperFeedManager(long value, SuperFeedManager.FeedReportMessageType mesgType,
+                long timestamp) {
             if (mesgService == null) {
                 waitTillMessageServiceIsUp();
             }
             String feedRep = feedId.getDataverse() + ":" + feedId.getFeedName() + ":" + feedId.getDatasetName();
-            String message = mesgType.name().toLowerCase() + "|" + feedRep + "|" + feedRuntimeType + "|" + partition
-                    + "|" + value + "|" + nodeId + "|" + timestamp + "|" + EOL;
+            String message = mesgType.name().toLowerCase() + FeedMessageService.MessageSeparator + feedRep
+                    + FeedMessageService.MessageSeparator + feedRuntimeType + FeedMessageService.MessageSeparator
+                    + partition + FeedMessageService.MessageSeparator + value + FeedMessageService.MessageSeparator
+                    + nodeId + FeedMessageService.MessageSeparator + timestamp + FeedMessageService.MessageSeparator
+                    + EOL;
             try {
                 mesgService.sendMessage(message);
             } catch (IOException ioe) {
-                ioe.printStackTrace();
                 if (LOGGER.isLoggable(Level.WARNING)) {
-                    LOGGER.warning("Unable to send feed report to SFM for feed " + feedId + " " + feedRuntimeType + "["
-                            + partition + "]");
+                    LOGGER.warning("Unable to send feed report to Super Feed Manager for feed " + feedId + " "
+                            + feedRuntimeType + "[" + partition + "]");
                 }
             }
         }
 
         private void waitTillMessageServiceIsUp() {
             while (mesgService == null) {
-                mesgService = FeedManager.INSTANCE.getFeedMessageService(feedId);
+                mesgService = feedManager.getFeedMessageService(feedId);
                 if (mesgService == null) {
                     try {
                         /**
@@ -267,7 +303,8 @@
         }
 
         public void deactivate() {
-            this.cancel();
+            // cancel the timer task to avoid future execution. 
+            cancel();
             collectThroughput = false;
         }
 
@@ -295,8 +332,9 @@
         writer.fail();
         if (healthMonitor != null && !healthMonitor.feedRuntimeType.equals(FeedRuntimeType.INGESTION)) {
             healthMonitor.deactivate();
+        } else {
+            healthMonitor.reset();
         }
-        healthMonitor.reset();
     }
 
     @Override
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedIntakeOperatorDescriptor.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedIntakeOperatorDescriptor.java
index 281a62f..859ed11 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedIntakeOperatorDescriptor.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedIntakeOperatorDescriptor.java
@@ -18,8 +18,11 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import edu.uci.ics.asterix.metadata.feeds.FeedRuntime.FeedRuntimeId;
-import edu.uci.ics.asterix.metadata.feeds.FeedRuntime.FeedRuntimeType;
+import edu.uci.ics.asterix.common.api.IAsterixAppRuntimeContext;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime.FeedRuntimeId;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime.FeedRuntimeType;
+import edu.uci.ics.asterix.common.feeds.IFeedManager;
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
@@ -52,6 +55,9 @@
     /** The adaptor factory that is used to create an instance of the feed adaptor **/
     private IAdapterFactory adapterFactory;
 
+    /** The (singleton) instance of IFeedManager **/
+    private IFeedManager feedManager;
+
     public FeedIntakeOperatorDescriptor(JobSpecification spec, FeedConnectionId feedId, IAdapterFactory adapterFactory,
             ARecordType atype, RecordDescriptor rDesc, Map<String, String> feedPolicy) {
         super(spec, 0, 1);
@@ -67,7 +73,10 @@
             throws HyracksDataException {
         IFeedAdapter adapter = null;
         FeedRuntimeId feedRuntimeId = new FeedRuntimeId(FeedRuntimeType.INGESTION, feedId, partition);
-        IngestionRuntime ingestionRuntime = (IngestionRuntime) FeedManager.INSTANCE.getFeedRuntime(feedRuntimeId);
+        IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
+                .getApplicationContext().getApplicationObject();
+        this.feedManager = runtimeCtx.getFeedManager();
+        IngestionRuntime ingestionRuntime = (IngestionRuntime) feedManager.getFeedRuntime(feedRuntimeId);
         try {
             if (ingestionRuntime == null) {
                 // create an instance of a feed adaptor to ingest data.
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedIntakeOperatorNodePushable.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedIntakeOperatorNodePushable.java
index 9a70ca3..3dbd2e2 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedIntakeOperatorNodePushable.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedIntakeOperatorNodePushable.java
@@ -20,8 +20,13 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import edu.uci.ics.asterix.common.api.IAsterixAppRuntimeContext;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime.FeedRuntimeType;
+import edu.uci.ics.asterix.common.feeds.FeedRuntimeManager;
+import edu.uci.ics.asterix.common.feeds.IFeedManager;
 import edu.uci.ics.asterix.metadata.feeds.AdapterRuntimeManager.State;
-import edu.uci.ics.asterix.metadata.feeds.FeedRuntime.FeedRuntimeType;
 import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
 import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
@@ -34,15 +39,17 @@
 
     private static Logger LOGGER = Logger.getLogger(FeedIntakeOperatorNodePushable.class.getName());
 
-    private IFeedAdapter adapter;
     private final int partition;
     private final FeedConnectionId feedId;
     private final LinkedBlockingQueue<IFeedMessage> inbox;
     private final Map<String, String> feedPolicy;
     private final FeedPolicyEnforcer policyEnforcer;
-    private FeedRuntime ingestionRuntime;
     private final String nodeId;
-    private FrameTupleAccessor fta;
+    private final FrameTupleAccessor fta;
+    private final IFeedManager feedManager;
+
+    private FeedRuntime ingestionRuntime;
+    private IFeedAdapter adapter;
     private FeedFrameWriter feedFrameWriter;
 
     public FeedIntakeOperatorNodePushable(IHyracksTaskContext ctx, FeedConnectionId feedId, IFeedAdapter adapter,
@@ -56,7 +63,9 @@
         policyEnforcer = new FeedPolicyEnforcer(feedId, feedPolicy);
         nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
         fta = new FrameTupleAccessor(ctx.getFrameSize(), recordDesc);
-
+        IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
+                .getApplicationContext().getApplicationObject();
+        this.feedManager = runtimeCtx.getFeedManager();
     }
 
     @Override
@@ -66,8 +75,9 @@
         try {
             if (ingestionRuntime == null) {
                 feedFrameWriter = new FeedFrameWriter(writer, this, feedId, policyEnforcer, nodeId,
-                        FeedRuntimeType.INGESTION, partition, fta);
-                adapterRuntimeMgr = new AdapterRuntimeManager(feedId, adapter, feedFrameWriter, partition, inbox);
+                        FeedRuntimeType.INGESTION, partition, fta, feedManager);
+                adapterRuntimeMgr = new AdapterRuntimeManager(feedId, adapter, feedFrameWriter, partition, inbox,
+                        feedManager);
 
                 if (adapter instanceof AbstractFeedDatasourceAdapter) {
                     ((AbstractFeedDatasourceAdapter) adapter).setFeedPolicyEnforcer(policyEnforcer);
@@ -96,7 +106,7 @@
                     adapterRuntimeMgr.wait();
                 }
             }
-            FeedManager.INSTANCE.deRegisterFeedRuntime(ingestionRuntime.getFeedRuntimeId());
+            feedManager.deRegisterFeedRuntime(ingestionRuntime.getFeedRuntimeId());
             feedFrameWriter.close();
         } catch (InterruptedException ie) {
             if (policyEnforcer.getFeedPolicyAccessor().continueOnHardwareFailure()) {
@@ -104,7 +114,7 @@
                     LOGGER.info("Continuing on failure as per feed policy, switching to INACTIVE INGESTION temporarily");
                 }
                 adapterRuntimeMgr.setState(State.INACTIVE_INGESTION);
-                FeedRuntimeManager runtimeMgr = FeedManager.INSTANCE.getFeedRuntimeManager(feedId);
+                FeedRuntimeManager runtimeMgr = feedManager.getFeedRuntimeManager(feedId);
                 try {
                     runtimeMgr.close(false);
                 } catch (IOException ioe) {
@@ -117,8 +127,8 @@
                 if (LOGGER.isLoggable(Level.INFO)) {
                     LOGGER.info("Interrupted Exception, something went wrong");
                 }
-               
-                FeedManager.INSTANCE.deRegisterFeedRuntime(ingestionRuntime.getFeedRuntimeId());
+
+                feedManager.deRegisterFeedRuntime(ingestionRuntime.getFeedRuntimeId());
                 feedFrameWriter.close();
                 throw new HyracksDataException(ie);
             }
@@ -132,33 +142,3 @@
         return feedPolicy;
     }
 }
-
-class FeedInboxMonitor extends Thread {
-
-    private LinkedBlockingQueue<IFeedMessage> inbox;
-    private final AdapterRuntimeManager runtimeMgr;
-
-    public FeedInboxMonitor(AdapterRuntimeManager runtimeMgr, LinkedBlockingQueue<IFeedMessage> inbox, int partition) {
-        this.inbox = inbox;
-        this.runtimeMgr = runtimeMgr;
-    }
-
-    @Override
-    public void run() {
-        while (true) {
-            try {
-                IFeedMessage feedMessage = inbox.take();
-                switch (feedMessage.getMessageType()) {
-                    case END:
-                        runtimeMgr.stop();
-                        break;
-                }
-            } catch (InterruptedException ie) {
-                break;
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        }
-    }
-
-}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedManager.java
index d910ca8..8b92994 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedManager.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedManager.java
@@ -20,21 +20,29 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import edu.uci.ics.asterix.metadata.feeds.FeedRuntime.FeedRuntimeId;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
+import edu.uci.ics.asterix.common.feeds.FeedMessageService;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime.FeedRuntimeId;
+import edu.uci.ics.asterix.common.feeds.FeedRuntimeManager;
+import edu.uci.ics.asterix.common.feeds.IFeedManager;
+import edu.uci.ics.asterix.common.feeds.SuperFeedManager;
 
+/**
+ * An implementation of the IFeedManager interface.
+ * Provider necessary central repository for registering/retrieving
+ * artifacts/services associated with a feed.
+ */
 public class FeedManager implements IFeedManager {
 
     private static final Logger LOGGER = Logger.getLogger(FeedManager.class.getName());
 
-    public static final long SOCKET_CONNECT_TIMEOUT = 5000;
-
-    public static FeedManager INSTANCE = new FeedManager();
-
-    private FeedManager() {
-
-    }
-
     private Map<FeedConnectionId, FeedRuntimeManager> feedRuntimeManagers = new HashMap<FeedConnectionId, FeedRuntimeManager>();
+    private final String nodeId;
+
+    public FeedManager(String nodeId) {
+        this.nodeId = nodeId;
+    }
 
     public FeedRuntimeManager getFeedRuntimeManager(FeedConnectionId feedId) {
         return feedRuntimeManagers.get(feedId);
@@ -45,6 +53,7 @@
         return mgr == null ? null : mgr.getExecutorService();
     }
 
+    @Override
     public FeedMessageService getFeedMessageService(FeedConnectionId feedId) {
         FeedRuntimeManager mgr = feedRuntimeManagers.get(feedId);
         return mgr == null ? null : mgr.getMessageService();
@@ -56,7 +65,7 @@
             FeedRuntimeManager mgr = feedRuntimeManagers.get(feedId);
             if (mgr == null) {
                 if (LOGGER.isLoggable(Level.WARNING)) {
-                    LOGGER.warning("unknown feed id: " + feedId);
+                    LOGGER.warning("Unknown feed id: " + feedId);
                 }
             } else {
                 if (LOGGER.isLoggable(Level.INFO)) {
@@ -81,7 +90,7 @@
         if (runtimeMgr == null) {
             synchronized (feedRuntimeManagers) {
                 if (runtimeMgr == null) {
-                    runtimeMgr = new FeedRuntimeManager(feedId);
+                    runtimeMgr = new FeedRuntimeManager(feedId, this);
                     feedRuntimeManagers.put(feedId, runtimeMgr);
                 }
             }
@@ -126,4 +135,9 @@
         FeedRuntimeManager runtimeMgr = feedRuntimeManagers.get(feedId);
         return runtimeMgr != null ? runtimeMgr.getSuperFeedManager() : null;
     }
+
+    @Override
+    public String toString() {
+        return "FeedManager " + "[" + nodeId + "]";
+    }
 }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedManagerElectMessage.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedManagerElectMessage.java
index aedd124..f3cc6a8 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedManagerElectMessage.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedManagerElectMessage.java
@@ -14,6 +14,8 @@
  */
 package edu.uci.ics.asterix.metadata.feeds;
 
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
+
 /**
  * A feed control message containing the altered values for
  * adapter configuration parameters. This message is dispatched
@@ -41,7 +43,7 @@
 
     @Override
     public String toString() {
-        return host + "_" + nodeId + "[" + port + "]";
+        return MessageType.SUPER_FEED_MANAGER_ELECT.name() + " " + host + "_" + nodeId + "[" + port + "]";
     }
 
     public String getHost() {
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessage.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessage.java
index 36fcf4b..019f21c 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessage.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessage.java
@@ -14,7 +14,7 @@
  */
 package edu.uci.ics.asterix.metadata.feeds;
 
-import edu.uci.ics.asterix.metadata.feeds.IFeedMessage;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
 
 /**
  * A control message that can be sent to the runtime instance of a
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageOperatorDescriptor.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageOperatorDescriptor.java
index fae2fe4..d33bf5e 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageOperatorDescriptor.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageOperatorDescriptor.java
@@ -14,6 +14,7 @@
  */
 package edu.uci.ics.asterix.metadata.feeds;
 
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
 import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
 import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
 import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageOperatorNodePushable.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageOperatorNodePushable.java
index a8541f5..47b00dd 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageOperatorNodePushable.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMessageOperatorNodePushable.java
@@ -17,8 +17,13 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import edu.uci.ics.asterix.metadata.feeds.FeedRuntime.FeedRuntimeId;
-import edu.uci.ics.asterix.metadata.feeds.FeedRuntime.FeedRuntimeType;
+import edu.uci.ics.asterix.common.api.IAsterixAppRuntimeContext;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime.FeedRuntimeId;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime.FeedRuntimeType;
+import edu.uci.ics.asterix.common.feeds.IFeedManager;
+import edu.uci.ics.asterix.common.feeds.SuperFeedManager;
 import edu.uci.ics.hyracks.api.application.INCApplicationContext;
 import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
@@ -35,6 +40,7 @@
     private final IFeedMessage feedMessage;
     private final int partition;
     private final IHyracksTaskContext ctx;
+    private final IFeedManager feedManager;
 
     public FeedMessageOperatorNodePushable(IHyracksTaskContext ctx, FeedConnectionId feedId, IFeedMessage feedMessage,
             int partition, int nPartitions) {
@@ -42,6 +48,9 @@
         this.feedMessage = feedMessage;
         this.partition = partition;
         this.ctx = ctx;
+        IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
+                .getApplicationContext().getApplicationObject();
+        this.feedManager = runtimeCtx.getFeedManager();
     }
 
     @Override
@@ -49,7 +58,7 @@
         try {
             writer.open();
             FeedRuntimeId runtimeId = new FeedRuntimeId(FeedRuntimeType.INGESTION, feedId, partition);
-            FeedRuntime feedRuntime = FeedManager.INSTANCE.getFeedRuntime(runtimeId);
+            FeedRuntime feedRuntime = feedManager.getFeedRuntime(runtimeId);
             boolean ingestionLocation = feedRuntime != null;
 
             switch (feedMessage.getMessageType()) {
@@ -70,12 +79,12 @@
 
                 case SUPER_FEED_MANAGER_ELECT:
                     if (LOGGER.isLoggable(Level.INFO)) {
-                        LOGGER.info("Registering SUPER Feed MGR for :" + feedId);
+                        LOGGER.info("Registering Supers Feed Manager for :" + feedId);
                     }
                     FeedManagerElectMessage mesg = ((FeedManagerElectMessage) feedMessage);
                     SuperFeedManager sfm = new SuperFeedManager(mesg.getFeedId(), mesg.getHost(), mesg.getNodeId(),
-                            mesg.getPort());
-                    synchronized (FeedManager.INSTANCE) {
+                            mesg.getPort(), feedManager);
+                    synchronized (feedManager) {
                         INCApplicationContext ncCtx = ctx.getJobletContext().getApplicationContext();
                         String nodeId = ncCtx.getNodeId();
                         if (sfm.getNodeId().equals(nodeId)) {
@@ -83,7 +92,7 @@
                         } else {
                             Thread.sleep(5000);
                         }
-                        FeedManager.INSTANCE.registerSuperFeedManager(feedId, sfm);
+                        feedManager.registerSuperFeedManager(feedId, sfm);
                         if (LOGGER.isLoggable(Level.INFO)) {
                             LOGGER.info("Registered super feed mgr " + sfm + " for feed " + feedId);
                         }
@@ -92,7 +101,6 @@
             }
 
         } catch (Exception e) {
-            e.printStackTrace();
             throw new HyracksDataException(e);
         } finally {
             writer.close();
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMetaOperatorDescriptor.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMetaOperatorDescriptor.java
index 3f8ab8b..1a8a460 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMetaOperatorDescriptor.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedMetaOperatorDescriptor.java
@@ -4,10 +4,14 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import edu.uci.ics.asterix.common.api.IAsterixAppRuntimeContext;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime.FeedRuntimeId;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime.FeedRuntimeState;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime.FeedRuntimeType;
+import edu.uci.ics.asterix.common.feeds.IFeedManager;
 import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
-import edu.uci.ics.asterix.metadata.feeds.FeedRuntime.FeedRuntimeId;
-import edu.uci.ics.asterix.metadata.feeds.FeedRuntime.FeedRuntimeState;
-import edu.uci.ics.asterix.metadata.feeds.FeedRuntime.FeedRuntimeType;
 import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
 import edu.uci.ics.hyracks.api.dataflow.IActivity;
 import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
@@ -124,6 +128,9 @@
         /** Allows to iterate over the tuples in a frame **/
         private FrameTupleAccessor fta;
 
+        /** The (singleton) instance of IFeedManager **/
+        private IFeedManager feedManager;
+
         public FeedMetaNodePushable(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider,
                 int partition, int nPartitions, IOperatorDescriptor coreOperator, FeedConnectionId feedConnectionId,
                 FeedPolicy feedPolicy, FeedRuntimeType runtimeType) throws HyracksDataException {
@@ -135,16 +142,19 @@
             this.feedId = feedConnectionId;
             this.nodeId = ctx.getJobletContext().getApplicationContext().getNodeId();
             fta = new FrameTupleAccessor(ctx.getFrameSize(), recordDesc);
+            IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
+                    .getApplicationContext().getApplicationObject();
+            this.feedManager = runtimeCtx.getFeedManager();
         }
 
         @Override
         public void open() throws HyracksDataException {
             FeedRuntimeId runtimeId = new FeedRuntimeId(runtimeType, feedId, partition);
             try {
-                feedRuntime = FeedManager.INSTANCE.getFeedRuntime(runtimeId);
+                feedRuntime = feedManager.getFeedRuntime(runtimeId);
                 if (feedRuntime == null) {
                     feedRuntime = new FeedRuntime(feedId, partition, runtimeType);
-                    FeedManager.INSTANCE.registerFeedRuntime(feedRuntime);
+                    feedManager.registerFeedRuntime(feedRuntime);
                     if (LOGGER.isLoggable(Level.WARNING)) {
                         LOGGER.warning("Did not find a saved state from a previous zombie, starting a new instance for "
                                 + runtimeType + " node.");
@@ -158,11 +168,10 @@
                     resumeOldState = true;
                 }
                 FeedFrameWriter mWriter = new FeedFrameWriter(writer, this, feedId, policyEnforcer, nodeId,
-                        runtimeType, partition, fta);
+                        runtimeType, partition, fta, feedManager);
                 coreOperatorNodePushable.setOutputFrameWriter(0, mWriter, recordDesc);
                 coreOperatorNodePushable.open();
             } catch (Exception e) {
-                e.printStackTrace();
                 if (LOGGER.isLoggable(Level.SEVERE)) {
                     LOGGER.severe("Unable to initialize feed operator " + feedRuntime + " [" + partition + "]");
                 }
@@ -182,27 +191,16 @@
                     feedRuntime.setRuntimeState(null);
                     resumeOldState = false;
                 }
+                currentBuffer = buffer;
                 coreOperatorNodePushable.nextFrame(buffer);
+                currentBuffer = null;
             } catch (HyracksDataException e) {
                 if (policyEnforcer.getFeedPolicyAccessor().continueOnApplicationFailure()) {
-                    boolean isExceptionHarmful = handleException(e.getCause());
+                    boolean isExceptionHarmful = e.getCause() instanceof TreeIndexException && !resumeOldState;
                     if (isExceptionHarmful) {
-                        // log the tuple
+                        // TODO: log the tuple
                         FeedRuntimeState runtimeState = new FeedRuntimeState(buffer, writer, e);
                         feedRuntime.setRuntimeState(runtimeState);
-                        String message = e.getMessage();
-                        String tIndexString = message.substring(message.lastIndexOf(':'));
-                        int tupleIndex = 0;
-                        if (tIndexString != null) {
-                            tupleIndex = Integer.parseInt(tIndexString);
-                        }
-                        fta.reset(buffer);
-                        int endOffset = fta.getTupleEndOffset(tupleIndex);
-                        buffer.flip();
-                        buffer.position(endOffset + 1);
-                        if (LOGGER.isLoggable(Level.WARNING)) {
-                            LOGGER.warning("Harmful exception (parked data) tupleIndex " + tupleIndex + e);
-                        }
                     } else {
                         // ignore the frame (exception is expected)
                         if (LOGGER.isLoggable(Level.WARNING)) {
@@ -218,23 +216,6 @@
             }
         }
 
-        private boolean handleException(Throwable exception) {
-            if (exception instanceof TreeIndexException) {
-                if (resumeOldState) {
-                    if (LOGGER.isLoggable(Level.WARNING)) {
-                        LOGGER.warning("Received duplicate key exception but that is possible post recovery");
-                    }
-                    return false;
-                } else {
-                    if (LOGGER.isLoggable(Level.SEVERE)) {
-                        LOGGER.severe("Received duplicate key exception!");
-                    }
-                    return true;
-                }
-            }
-            return true;
-        }
-
         @Override
         public void fail() throws HyracksDataException {
             if (LOGGER.isLoggable(Level.WARNING)) {
@@ -248,10 +229,9 @@
                         LOGGER.warning("Saved feed compute runtime for revivals" + feedRuntime.getFeedRuntimeId());
                     }
                 } else {
-                    FeedManager.INSTANCE.deRegisterFeedRuntime(feedRuntime.getFeedRuntimeId());
+                    feedManager.deRegisterFeedRuntime(feedRuntime.getFeedRuntimeId());
                     if (LOGGER.isLoggable(Level.WARNING)) {
-                        LOGGER.warning(" No state to save, de-registered feed compute runtime "
-                                + feedRuntime.getFeedRuntimeId());
+                        LOGGER.warning("No state to save, de-registered feed runtime " + feedRuntime.getFeedRuntimeId());
                     }
                 }
             }
@@ -261,7 +241,7 @@
         @Override
         public void close() throws HyracksDataException {
             coreOperatorNodePushable.close();
-            FeedManager.INSTANCE.deRegisterFeedRuntime(feedRuntime.getFeedRuntimeId());
+            feedManager.deRegisterFeedRuntime(feedRuntime.getFeedRuntimeId());
         }
 
     }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedPolicyAccessor.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedPolicyAccessor.java
index a3c505b..fd9716c 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedPolicyAccessor.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedPolicyAccessor.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package edu.uci.ics.asterix.metadata.feeds;
 
 import java.util.Map;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedPolicyEnforcer.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedPolicyEnforcer.java
index b94e228..44487ec 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedPolicyEnforcer.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedPolicyEnforcer.java
@@ -1,11 +1,25 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package edu.uci.ics.asterix.metadata.feeds;
 
 import java.rmi.RemoteException;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 import edu.uci.ics.asterix.common.exceptions.ACIDException;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
 import edu.uci.ics.asterix.metadata.MetadataManager;
 import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
 import edu.uci.ics.asterix.metadata.entities.FeedActivity;
@@ -24,7 +38,7 @@
                 null, new HashMap<String, String>());
     }
 
-    public boolean handleSoftwareFailure(Exception e) throws RemoteException, ACIDException {
+    public boolean continueIngestionPostSoftwareFailure(Exception e) throws RemoteException, ACIDException {
         boolean continueIngestion = feedPolicyAccessor.continueOnApplicationFailure();
         if (feedPolicyAccessor.logErrorOnFailure()) {
             persistExceptionDetails(e);
@@ -49,10 +63,6 @@
         }
     }
 
-    public void handleHardwareFailure(List<String> nodeId) {
-
-    }
-
     public FeedPolicyAccessor getFeedPolicyAccessor() {
         return feedPolicyAccessor;
     }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedReport.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedReport.java
index fbff931..d3225a2 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedReport.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedReport.java
@@ -3,8 +3,9 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import edu.uci.ics.asterix.metadata.feeds.FeedRuntime.FeedRuntimeType;
-import edu.uci.ics.asterix.metadata.feeds.SuperFeedManager.FeedReportMessageType;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime.FeedRuntimeType;
+import edu.uci.ics.asterix.common.feeds.SuperFeedManager.FeedReportMessageType;
 
 public class FeedReport implements Comparable {
 
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedSourceProperties.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedSourceProperties.java
deleted file mode 100644
index 268e30a..0000000
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedSourceProperties.java
+++ /dev/null
@@ -1,46 +0,0 @@
-package edu.uci.ics.asterix.metadata.feeds;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class FeedSourceProperties {
-
-    public enum ExchangeMode {
-        PULL,
-        PUSH
-    }
-
-    public enum IngestionRuntimeType {
-        STATELESS,
-        STATEFULL
-    }
-
-    public enum ComputeRuntimeType {
-        STATELESS,
-        STATEFULL
-    }
-
-    public static class FeedSourcePropertyKeys {
-        public static final String EXCHANGE_MODE = "exchange_mode";
-        public static final String INGESTION_RUNTIME_TYPE = "ingestion_runtime_type";
-        public static final String COMPUTE_RUNTIME_TYPE = "compute_runtime_type";
-        public static final String BACKWARD_TIME_TRAVEL = "backward_time_travel";
-        public static final String COMPUTE_IDEMPOTENCE = "compute_idempotence";
-    }
-
-    private Map<String, String> sourceConfiguration;
-
-    public FeedSourceProperties(Map<String, String> sourceConfiguration) {
-        this.sourceConfiguration = sourceConfiguration;
-    }
-
-    public static class FeedSourcePropertyAccessor {
-
-        private boolean computeIdempotence;
-        private boolean backwardTimeTravel;
-        private ExchangeMode exchangeMode;
-        private IngestionRuntimeType ingestionRuntimeType;
-        private ComputeRuntimeType computeRuntimeType;
-
-    }
-}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedUtil.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedUtil.java
index 177c082..d3693bf 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedUtil.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/FeedUtil.java
@@ -23,16 +23,19 @@
 import org.apache.commons.lang3.tuple.Pair;
 
 import edu.uci.ics.asterix.common.dataflow.AsterixLSMTreeInsertDeleteOperatorDescriptor;
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime.FeedRuntimeType;
 import edu.uci.ics.asterix.metadata.MetadataManager;
 import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
 import edu.uci.ics.asterix.metadata.bootstrap.MetadataConstants;
 import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
 import edu.uci.ics.asterix.metadata.entities.DatasourceAdapter;
+import edu.uci.ics.asterix.metadata.entities.DatasourceAdapter.AdapterType;
 import edu.uci.ics.asterix.metadata.entities.Feed;
 import edu.uci.ics.asterix.metadata.entities.FeedActivity;
 import edu.uci.ics.asterix.metadata.entities.FeedActivity.FeedActivityType;
 import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
-import edu.uci.ics.asterix.metadata.feeds.FeedRuntime.FeedRuntimeType;
+import edu.uci.ics.asterix.metadata.functions.ExternalLibraryManager;
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
@@ -206,9 +209,24 @@
             adapterName = feed.getAdaptorName();
             adapterEntity = MetadataManager.INSTANCE.getAdapter(mdTxnCtx, MetadataConstants.METADATA_DATAVERSE_NAME,
                     adapterName);
+            if (adapterEntity == null) {
+                adapterEntity = MetadataManager.INSTANCE.getAdapter(mdTxnCtx, feed.getDataverseName(), adapterName);
+            }
+
             if (adapterEntity != null) {
                 adapterFactoryClassname = adapterEntity.getClassname();
-                adapterFactory = (IAdapterFactory) Class.forName(adapterFactoryClassname).newInstance();
+                switch (adapterEntity.getType()) {
+                    case INTERNAL:
+                        adapterFactory = (IAdapterFactory) Class.forName(adapterFactoryClassname).newInstance();
+                        break;
+                    case EXTERNAL:
+                        String[] anameComponents = adapterName.split("#");
+                        String libraryName = anameComponents[0];
+                        ClassLoader cl = ExternalLibraryManager.getLibraryClassLoader(feed.getDataverseName(),
+                                libraryName);
+                        adapterFactory = (IAdapterFactory) cl.loadClass(adapterFactoryClassname).newInstance();
+                        break;
+                }
             } else {
                 adapterFactoryClassname = AqlMetadataProvider.adapterFactoryMapping.get(adapterName);
                 if (adapterFactoryClassname != null) {
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IAdapterExecutor.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IAdapterExecutor.java
index 0d1f955..a03bf2b 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IAdapterExecutor.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IAdapterExecutor.java
@@ -1,5 +1,21 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package edu.uci.ics.asterix.metadata.feeds;
 
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
+
 public interface IAdapterExecutor {
 
     /**
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IDatasourceAdapter.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IDatasourceAdapter.java
index d0f8cdd..a4c5de9 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IDatasourceAdapter.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IDatasourceAdapter.java
@@ -30,13 +30,7 @@
      * An adapter can be used to read from an external data source and may also
      * allow writing to the external data source. This enum type indicates the
      * kind of operations supported by the adapter.
-     * 
-     * @caller Compiler uses this method to assert the validity of an operation
-     *         on an external dataset. The type of adapter associated with an
-     *         external dataset determines the set of valid operations allowed
-     *         on the dataset.
      */
-   
 
     /**
      * Triggers the adapter to begin ingesting data from the external source.
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IGenericAdapterFactory.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IGenericAdapterFactory.java
index ac0c711..54613d0 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IGenericAdapterFactory.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IGenericAdapterFactory.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package edu.uci.ics.asterix.metadata.feeds;
 
 import java.util.Map;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IngestionRuntime.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IngestionRuntime.java
index 37d49b4..0ea0d38 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IngestionRuntime.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/IngestionRuntime.java
@@ -14,6 +14,9 @@
  */
 package edu.uci.ics.asterix.metadata.feeds;
 
+import edu.uci.ics.asterix.common.feeds.FeedConnectionId;
+import edu.uci.ics.asterix.common.feeds.FeedRuntime;
+
 public class IngestionRuntime extends FeedRuntime {
 
     private AdapterRuntimeManager adapterRuntimeManager;
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/MessageListener.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/MessageListener.java
index 20a6b0e..1bd042a 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/MessageListener.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/MessageListener.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package edu.uci.ics.asterix.metadata.feeds;
 
 import java.io.IOException;
@@ -15,7 +29,7 @@
 
     private static final Logger LOGGER = Logger.getLogger(MessageListener.class.getName());
 
-    private int port;
+    private final int port;
     private final LinkedBlockingQueue<String> outbox;
 
     private ExecutorService executorService = Executors.newFixedThreadPool(10);
@@ -29,7 +43,9 @@
 
     public void stop() {
         listenerServer.stop();
-        System.out.println("STOPPED MESSAGE RECEIVING SERVICE AT " + port);
+        if (LOGGER.isLoggable(Level.INFO)) {
+            LOGGER.info("Stopped message service at " + port);
+        }
         if (!executorService.isShutdown()) {
             executorService.shutdownNow();
         }
@@ -37,9 +53,11 @@
     }
 
     public void start() throws IOException {
-        System.out.println("STARTING MESSAGE RECEIVING SERVICE AT " + port);
         listenerServer = new MessageListenerServer(port, outbox);
         executorService.execute(listenerServer);
+        if (LOGGER.isLoggable(Level.INFO)) {
+            LOGGER.info("Starting message service at " + port);
+        }
     }
 
     private static class MessageListenerServer implements Runnable {
@@ -107,57 +125,8 @@
 
     }
 
-    private static class MessageParser implements Runnable {
-
-        private Socket client;
-        private IMessageAnalyzer messageAnalyzer;
-        private static final char EOL = (char) "\n".getBytes()[0];
-
-        public MessageParser(Socket client, IMessageAnalyzer messageAnalyzer) {
-            this.client = client;
-            this.messageAnalyzer = messageAnalyzer;
-        }
-
-        @Override
-        public void run() {
-            CharBuffer buffer = CharBuffer.allocate(5000);
-            char ch;
-            try {
-                InputStream in = client.getInputStream();
-                while (true) {
-                    ch = (char) in.read();
-                    if (((int) ch) == -1) {
-                        break;
-                    }
-                    while (ch != EOL) {
-                        buffer.put(ch);
-                        ch = (char) in.read();
-                    }
-                    buffer.flip();
-                    String s = new String(buffer.array());
-                    synchronized (messageAnalyzer) {
-                        messageAnalyzer.getMessageQueue().add(s + "\n");
-                    }
-                    buffer.position(0);
-                    buffer.limit(5000);
-                }
-            } catch (IOException ioe) {
-                ioe.printStackTrace();
-            } finally {
-                try {
-                    client.close();
-                } catch (IOException ioe) {
-                    // do nothing
-                }
-            }
-        }
-    }
-
     public static interface IMessageAnalyzer {
 
-        /**
-         * @return
-         */
         public LinkedBlockingQueue<String> getMessageQueue();
 
     }
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/RemoteSocketMessageListener.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/RemoteSocketMessageListener.java
index 822c638..d7e2e2f 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/RemoteSocketMessageListener.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/feeds/RemoteSocketMessageListener.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package edu.uci.ics.asterix.metadata.feeds;
 
 import java.io.IOException;
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/ExternalLibraryManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/functions/ExternalLibraryManager.java
similarity index 79%
rename from asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/ExternalLibraryManager.java
rename to asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/functions/ExternalLibraryManager.java
index 520020d..aa506d0 100755
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/library/ExternalLibraryManager.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/functions/ExternalLibraryManager.java
@@ -12,7 +12,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.asterix.external.library;
+package edu.uci.ics.asterix.metadata.functions;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -22,17 +22,17 @@
     private static Map<String, ClassLoader> libraryClassLoaders = new HashMap<String, ClassLoader>();
 
     public static void registerLibraryClassLoader(String dataverseName, String libraryName, ClassLoader classLoader) {
-        String key = dataverseName + "." + libraryName;
+        String key = getKey(dataverseName, libraryName);
         synchronized (libraryClassLoaders) {
             if (libraryClassLoaders.get(dataverseName) != null) {
-                throw new IllegalStateException("library class loader already registered!");
+                throw new IllegalStateException("Library class loader already registered!");
             }
             libraryClassLoaders.put(key, classLoader);
         }
     }
 
     public static void deregisterLibraryClassLoader(String dataverseName, String libraryName) {
-        String key = dataverseName + "." + libraryName;
+        String key = getKey(dataverseName, libraryName);
         synchronized (libraryClassLoaders) {
             if (libraryClassLoaders.get(dataverseName) != null) {
                 libraryClassLoaders.remove(key);
@@ -41,10 +41,14 @@
     }
 
     public static ClassLoader getLibraryClassLoader(String dataverseName, String libraryName) {
-        String key = dataverseName + "." + libraryName;
+        String key = getKey(dataverseName, libraryName);
         synchronized (libraryClassLoaders) {
             return libraryClassLoaders.get(key);
         }
     }
 
+    private static String getKey(String dataverseName, String libraryName) {
+        return dataverseName + "." + libraryName;
+    }
+
 }
\ No newline at end of file
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/AdmDataGen.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/AdmDataGen.java
index 333d4a3..eb85bb9 100644
--- a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/AdmDataGen.java
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/AdmDataGen.java
@@ -49,8 +49,9 @@
 import edu.uci.ics.asterix.common.annotations.RecordDataGenAnnotation;
 import edu.uci.ics.asterix.common.annotations.TypeDataGen;
 import edu.uci.ics.asterix.common.annotations.UndeclaredFieldsDataGen;
-import edu.uci.ics.asterix.common.exceptions.AsterixException;
 import edu.uci.ics.asterix.common.exceptions.ACIDException;
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.common.transactions.JobId;
 import edu.uci.ics.asterix.metadata.MetadataException;
 import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
 import edu.uci.ics.asterix.om.types.ARecordType;
@@ -61,7 +62,6 @@
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.asterix.om.types.TypeSignature;
 import edu.uci.ics.asterix.tools.translator.ADGenDmlTranslator;
-import edu.uci.ics.asterix.common.transactions.JobId;
 import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
 import edu.uci.ics.hyracks.algebricks.data.utils.WriteValueTools;
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/DataGenerator.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/DataGenerator.java
index 0c93564..7c18670 100644
--- a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/DataGenerator.java
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/DataGenerator.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package edu.uci.ics.asterix.tools.external.data;
 
 import java.io.BufferedReader;
@@ -7,14 +21,11 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.nio.CharBuffer;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Random;
-import java.util.UUID;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
@@ -33,227 +44,50 @@
 
 public class DataGenerator {
 
-    private static RandomDateGenerator randDateGen;
-    private static RandomNameGenerator randNameGen;
-    private static RandomEmploymentGenerator randEmpGen;
-    private static RandomMessageGenerator randMessageGen;
-    private static RandomLocationGenerator randLocationGen;
+    private RandomDateGenerator randDateGen;
+    private RandomNameGenerator randNameGen;
+    private RandomEmploymentGenerator randEmpGen;
+    private RandomMessageGenerator randMessageGen;
+    private RandomLocationGenerator randLocationGen;
 
-    private static DistributionHandler fbDistHandler;
-    private static DistributionHandler twDistHandler;
+    private DistributionHandler fbDistHandler;
+    private DistributionHandler twDistHandler;
 
-    private static int totalFbMessages;
-    private static int numFbOnlyUsers;
-    private static int totalTwMessages;
-    private static int numTwOnlyUsers;
+    private int totalFbMessages;
+    private int numFbOnlyUsers;
+    private int totalTwMessages;
+    private int numTwOnlyUsers;
 
-    private static int numCommonUsers;
+    private int numCommonUsers;
 
-    private static int fbUserId;
-    private static int twUserId;
+    private int fbUserId;
+    private int twUserId;
 
-    private static int fbMessageId;
-    private static int twMessageId;
+    private int fbMessageId;
+    private int twMessageId;
 
-    private static Random random = new Random();
+    private Random random = new Random();
 
-    private static String commonUserFbSuffix = "_fb";
-    private static String commonUserTwSuffix = "_tw";
+    private String commonUserFbSuffix = "_fb";
+    private String commonUserTwSuffix = "_tw";
 
-    private static String outputDir;
+    private String outputDir;
 
-    private static PartitionConfiguration partition;
+    private PartitionConfiguration partition;
 
-    private static FacebookUser fbUser = new FacebookUser();
-    private static TwitterUser twUser = new TwitterUser();
+    private FacebookUser fbUser = new FacebookUser();
+    private TwitterUser twUser = new TwitterUser();
 
-    private static FacebookMessage fbMessage = new FacebookMessage();
-    private static TweetMessage twMessage = new TweetMessage();
+    private FacebookMessage fbMessage = new FacebookMessage();
+    private TweetMessage twMessage = new TweetMessage();
 
-    private static int duration;
+    private int duration;
 
-    private static DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
-
-    private static void generateFacebookOnlyUsers(int numFacebookUsers) throws IOException {
-        FileAppender appender = FileUtil.getFileAppender(outputDir + "/" + "fb_users.adm", true, true);
-        FileAppender messageAppender = FileUtil.getFileAppender(outputDir + "/" + "fb_message.adm", true, true);
-
-        for (int i = 0; i < numFacebookUsers; i++) {
-            getFacebookUser(null);
-            appender.appendToFile(fbUser.toString());
-            generateFacebookMessages(fbUser, messageAppender, -1);
-        }
-        appender.close();
-        messageAppender.close();
-    }
-
-    private static void generateTwitterOnlyUsers(int numTwitterUsers) throws IOException {
-        FileAppender appender = FileUtil.getFileAppender(outputDir + "/" + "tw_users.adm", true, true);
-        FileAppender messageAppender = FileUtil.getFileAppender(outputDir + "/" + "tw_message.adm", true, true);
-
-        for (int i = 0; i < numTwitterUsers; i++) {
-            getTwitterUser(null);
-            appender.appendToFile(twUser.toString());
-            generateTwitterMessages(twUser, messageAppender, -1);
-        }
-        appender.close();
-        messageAppender.close();
-    }
-
-    private static void generateCommonUsers() throws IOException {
-        FileAppender fbAppender = FileUtil.getFileAppender(outputDir + "/" + "fb_users.adm", true, false);
-        FileAppender twAppender = FileUtil.getFileAppender(outputDir + "/" + "tw_users.adm", true, false);
-        FileAppender fbMessageAppender = FileUtil.getFileAppender(outputDir + "/" + "fb_message.adm", true, false);
-        FileAppender twMessageAppender = FileUtil.getFileAppender(outputDir + "/" + "tw_message.adm", true, false);
-
-        for (int i = 0; i < numCommonUsers; i++) {
-            getFacebookUser(commonUserFbSuffix);
-            fbAppender.appendToFile(fbUser.toString());
-            generateFacebookMessages(fbUser, fbMessageAppender, -1);
-
-            getCorrespondingTwitterUser(fbUser);
-            twAppender.appendToFile(twUser.toString());
-            generateTwitterMessages(twUser, twMessageAppender, -1);
-        }
-
-        fbAppender.close();
-        twAppender.close();
-        fbMessageAppender.close();
-        twMessageAppender.close();
-    }
-
-    private static void generateFacebookMessages(FacebookUser user, FileAppender appender, int numMsg)
-            throws IOException {
-        Message message;
-        int numMessages = 0;
-        if (numMsg == -1) {
-            numMessages = fbDistHandler
-                    .getFromDistribution(fbUserId - partition.getTargetPartition().getFbUserKeyMin());
-        }
-        for (int i = 0; i < numMessages; i++) {
-            message = randMessageGen.getNextRandomMessage();
-            Point location = randLocationGen.getRandomPoint();
-            fbMessage.reset(fbMessageId++, user.getId(), random.nextInt(totalFbMessages + 1), location, message);
-            appender.appendToFile(fbMessage.toString());
-        }
-    }
-
-    private static void generateTwitterMessages(TwitterUser user, FileAppender appender, int numMsg) throws IOException {
-        Message message;
-        int numMessages = 0;
-        if (numMsg == -1) {
-            numMessages = twDistHandler
-                    .getFromDistribution(twUserId - partition.getTargetPartition().getTwUserKeyMin());
-        }
-
-        for (int i = 0; i < numMessages; i++) {
-            message = randMessageGen.getNextRandomMessage();
-            Point location = randLocationGen.getRandomPoint();
-            DateTime sendTime = randDateGen.getNextRandomDatetime();
-            twMessage.reset(twMessageId + "", user, location, sendTime, message.getReferredTopics(), message);
-            twMessageId++;
-            appender.appendToFile(twMessage.toString());
-        }
-    }
-
-    public static Iterator<TweetMessage> getTwitterMessageIterator() {
-        return new TweetMessageIterator(duration);
-    }
-
-    public static class TweetMessageIterator implements Iterator<TweetMessage> {
-
-        private final int duration;
-        private long startTime = 0;
-
-        public TweetMessageIterator(int duration) {
-            this.duration = duration;
-        }
-
-        @Override
-        public boolean hasNext() {
-            if (startTime == 0) {
-                startTime = System.currentTimeMillis();
-            }
-            return System.currentTimeMillis() - startTime < duration * 1000;
-        }
-
-        @Override
-        public TweetMessage next() {
-            getTwitterUser(null);
-            Message message = randMessageGen.getNextRandomMessage();
-            Point location = randLocationGen.getRandomPoint();
-            DateTime sendTime = randDateGen.getNextRandomDatetime();
-            twMessage.reset(UUID.randomUUID().toString(), twUser, location, sendTime, message.getReferredTopics(),
-                    message);
-            twMessageId++;
-            if (twUserId > numTwOnlyUsers) {
-                twUserId = 1;
-            }
-            return twMessage;
-
-        }
-
-        @Override
-        public void remove() {
-            // TODO Auto-generated method stub
-
-        }
-
-    }
-
-    public static class InitializationInfo {
-        public Date startDate = new Date(1, 1, 2005);
-        public Date endDate = new Date(8, 20, 2012);
-        public String[] lastNames = DataGenerator.lastNames;
-        public String[] firstNames = DataGenerator.firstNames;
-        public String[] vendors = DataGenerator.vendors;
-        public String[] jargon = DataGenerator.jargon;
-        public String[] org_list = DataGenerator.org_list;
-        public int percentEmployed = 90;
-        public Date employmentStartDate = new Date(1, 1, 2000);
-        public Date employmentEndDate = new Date(31, 12, 2012);
-        public int totalFbMessages;
-        public int numFbOnlyUsers;
-        public int totalTwMessages;
-        public int numTwOnlyUsers = 5000;
-        public int numCommonUsers;
-        public int fbUserIdMin;
-        public int fbMessageIdMin;
-        public int twUserIdMin;
-        public int twMessageIdMin;
-        public int timeDurationInSecs = 60;
-
-    }
-
-    public static void initialize(InitializationInfo info) {
-        randDateGen = new RandomDateGenerator(info.startDate, info.endDate);
-        randNameGen = new RandomNameGenerator(info.firstNames, info.lastNames);
-        randEmpGen = new RandomEmploymentGenerator(info.percentEmployed, info.employmentStartDate,
-                info.employmentEndDate, info.org_list);
-        randLocationGen = new RandomLocationGenerator(24, 49, 66, 98);
-        randMessageGen = new RandomMessageGenerator(info.vendors, info.jargon);
-        fbDistHandler = new DistributionHandler(info.totalFbMessages, 0.5, info.numFbOnlyUsers + info.numCommonUsers);
-        twDistHandler = new DistributionHandler(info.totalTwMessages, 0.5, info.numTwOnlyUsers + info.numCommonUsers);
-        fbUserId = info.fbUserIdMin;
-        twUserId = info.twUserIdMin;
-
-        fbMessageId = info.fbMessageIdMin;
-        twMessageId = info.fbMessageIdMin;
-        duration = info.timeDurationInSecs;
-    }
-
-    public static void main(String args[]) throws Exception {
-
-        String controllerInstallDir = null;
-        if (args.length < 2) {
-            printUsage();
-            System.exit(1);
-        } else {
-            controllerInstallDir = args[0];
-            String partitionConfXML = controllerInstallDir + "/output/partition-conf.xml";
-            String partitionName = args[1];
-            partition = XMLUtil.getPartitionConfiguration(partitionConfXML, partitionName);
-        }
+    public DataGenerator(String[] args) throws Exception {
+        String controllerInstallDir = args[0];
+        String partitionConfXML = controllerInstallDir + "/output/partition-conf.xml";
+        String partitionName = args[1];
+        partition = XMLUtil.getPartitionConfiguration(partitionConfXML, partitionName);
 
         // 1
         randDateGen = new RandomDateGenerator(new Date(1, 1, 2005), new Date(8, 20, 2012));
@@ -292,7 +126,187 @@
         twMessageId = partition.getTargetPartition().getTwMessageIdMin();
 
         outputDir = partition.getSourcePartition().getPath();
-        generateData();
+    }
+
+    public DataGenerator(InitializationInfo info) {
+        initialize(info);
+    }
+
+    private void generateFacebookOnlyUsers(int numFacebookUsers) throws IOException {
+        FileAppender appender = FileUtil.getFileAppender(outputDir + "/" + "fb_users.adm", true, true);
+        FileAppender messageAppender = FileUtil.getFileAppender(outputDir + "/" + "fb_message.adm", true, true);
+
+        for (int i = 0; i < numFacebookUsers; i++) {
+            getFacebookUser(null);
+            appender.appendToFile(fbUser.toString());
+            generateFacebookMessages(fbUser, messageAppender, -1);
+        }
+        appender.close();
+        messageAppender.close();
+    }
+
+    private void generateTwitterOnlyUsers(int numTwitterUsers) throws IOException {
+        FileAppender appender = FileUtil.getFileAppender(outputDir + "/" + "tw_users.adm", true, true);
+        FileAppender messageAppender = FileUtil.getFileAppender(outputDir + "/" + "tw_message.adm", true, true);
+
+        for (int i = 0; i < numTwitterUsers; i++) {
+            getTwitterUser(null);
+            appender.appendToFile(twUser.toString());
+            generateTwitterMessages(twUser, messageAppender, -1);
+        }
+        appender.close();
+        messageAppender.close();
+    }
+
+    private void generateCommonUsers() throws IOException {
+        FileAppender fbAppender = FileUtil.getFileAppender(outputDir + "/" + "fb_users.adm", true, false);
+        FileAppender twAppender = FileUtil.getFileAppender(outputDir + "/" + "tw_users.adm", true, false);
+        FileAppender fbMessageAppender = FileUtil.getFileAppender(outputDir + "/" + "fb_message.adm", true, false);
+        FileAppender twMessageAppender = FileUtil.getFileAppender(outputDir + "/" + "tw_message.adm", true, false);
+
+        for (int i = 0; i < numCommonUsers; i++) {
+            getFacebookUser(commonUserFbSuffix);
+            fbAppender.appendToFile(fbUser.toString());
+            generateFacebookMessages(fbUser, fbMessageAppender, -1);
+
+            getCorrespondingTwitterUser(fbUser);
+            twAppender.appendToFile(twUser.toString());
+            generateTwitterMessages(twUser, twMessageAppender, -1);
+        }
+
+        fbAppender.close();
+        twAppender.close();
+        fbMessageAppender.close();
+        twMessageAppender.close();
+    }
+
+    private void generateFacebookMessages(FacebookUser user, FileAppender appender, int numMsg) throws IOException {
+        Message message;
+        int numMessages = 0;
+        if (numMsg == -1) {
+            numMessages = fbDistHandler
+                    .getFromDistribution(fbUserId - partition.getTargetPartition().getFbUserKeyMin());
+        }
+        for (int i = 0; i < numMessages; i++) {
+            message = randMessageGen.getNextRandomMessage();
+            Point location = randLocationGen.getRandomPoint();
+            fbMessage.reset(fbMessageId++, user.getId(), random.nextInt(totalFbMessages + 1), location, message);
+            appender.appendToFile(fbMessage.toString());
+        }
+    }
+
+    private void generateTwitterMessages(TwitterUser user, FileAppender appender, int numMsg) throws IOException {
+        Message message;
+        int numMessages = 0;
+        if (numMsg == -1) {
+            numMessages = twDistHandler
+                    .getFromDistribution(twUserId - partition.getTargetPartition().getTwUserKeyMin());
+        }
+
+        for (int i = 0; i < numMessages; i++) {
+            message = randMessageGen.getNextRandomMessage();
+            Point location = randLocationGen.getRandomPoint();
+            DateTime sendTime = randDateGen.getNextRandomDatetime();
+            twMessage.reset(twMessageId, user, location, sendTime, message.getReferredTopics(), message);
+            twMessageId++;
+            appender.appendToFile(twMessage.toString());
+        }
+    }
+
+    public Iterator<TweetMessage> getTwitterMessageIterator(int partition, byte seed) {
+        return new TweetMessageIterator(duration, partition, seed);
+    }
+
+    public class TweetMessageIterator implements Iterator<TweetMessage> {
+
+        private final int duration;
+        private long startTime = 0;
+        private final GULongIDGenerator idGen;
+
+        public TweetMessageIterator(int duration, int partition, byte seed) {
+            this.duration = duration;
+            this.idGen = new GULongIDGenerator(partition, seed);
+        }
+
+        @Override
+        public boolean hasNext() {
+            if (startTime == 0) {
+                startTime = System.currentTimeMillis();
+            }
+            return System.currentTimeMillis() - startTime < duration * 1000;
+        }
+
+        @Override
+        public TweetMessage next() {
+            getTwitterUser(null);
+            Message message = randMessageGen.getNextRandomMessage();
+            Point location = randLocationGen.getRandomPoint();
+            DateTime sendTime = randDateGen.getNextRandomDatetime();
+            twMessage.reset(idGen.getNextULong(), twUser, location, sendTime, message.getReferredTopics(), message);
+            twMessageId++;
+            if (twUserId > numTwOnlyUsers) {
+                twUserId = 1;
+            }
+            return twMessage;
+
+        }
+
+        @Override
+        public void remove() {
+            // TODO Auto-generated method stub
+        }
+
+    }
+
+    public static class InitializationInfo {
+        public Date startDate = new Date(1, 1, 2005);
+        public Date endDate = new Date(8, 20, 2012);
+        public String[] lastNames = DataGenerator.lastNames;
+        public String[] firstNames = DataGenerator.firstNames;
+        public String[] vendors = DataGenerator.vendors;
+        public String[] jargon = DataGenerator.jargon;
+        public String[] org_list = DataGenerator.org_list;
+        public int percentEmployed = 90;
+        public Date employmentStartDate = new Date(1, 1, 2000);
+        public Date employmentEndDate = new Date(31, 12, 2012);
+        public int totalFbMessages;
+        public int numFbOnlyUsers;
+        public int totalTwMessages;
+        public int numTwOnlyUsers = 5000;
+        public int numCommonUsers;
+        public int fbUserIdMin;
+        public int fbMessageIdMin;
+        public int twUserIdMin;
+        public int twMessageIdMin;
+        public int timeDurationInSecs = 60;
+
+    }
+
+    public void initialize(InitializationInfo info) {
+        randDateGen = new RandomDateGenerator(info.startDate, info.endDate);
+        randNameGen = new RandomNameGenerator(info.firstNames, info.lastNames);
+        randEmpGen = new RandomEmploymentGenerator(info.percentEmployed, info.employmentStartDate,
+                info.employmentEndDate, info.org_list);
+        randLocationGen = new RandomLocationGenerator(24, 49, 66, 98);
+        randMessageGen = new RandomMessageGenerator(info.vendors, info.jargon);
+        fbDistHandler = new DistributionHandler(info.totalFbMessages, 0.5, info.numFbOnlyUsers + info.numCommonUsers);
+        twDistHandler = new DistributionHandler(info.totalTwMessages, 0.5, info.numTwOnlyUsers + info.numCommonUsers);
+        fbUserId = info.fbUserIdMin;
+        twUserId = info.twUserIdMin;
+
+        fbMessageId = info.fbMessageIdMin;
+        twMessageId = info.twMessageIdMin;
+        duration = info.timeDurationInSecs;
+    }
+
+    public static void main(String args[]) throws Exception {
+        if (args.length < 2) {
+            printUsage();
+            System.exit(1);
+        }
+
+        DataGenerator dataGenerator = new DataGenerator(args);
+        dataGenerator.generateData();
     }
 
     public static void printUsage() {
@@ -300,14 +314,14 @@
         System.out.println(" Usage :" + " DataGenerator <path to configuration file> <partition name> ");
     }
 
-    public static void generateData() throws IOException {
+    public void generateData() throws IOException {
         generateFacebookOnlyUsers(numFbOnlyUsers);
         generateTwitterOnlyUsers(numTwOnlyUsers);
         generateCommonUsers();
         System.out.println("Partition :" + partition.getTargetPartition().getName() + " finished");
     }
 
-    public static void getFacebookUser(String usernameSuffix) {
+    public void getFacebookUser(String usernameSuffix) {
         String suggestedName = randNameGen.getRandomName();
         String[] nameComponents = suggestedName.split(" ");
         String name = nameComponents[0] + nameComponents[1];
@@ -322,7 +336,7 @@
         fbUser.reset(fbUserId++, alias, name, userSince, friendIds, emp);
     }
 
-    public static void getTwitterUser(String usernameSuffix) {
+    public void getTwitterUser(String usernameSuffix) {
         String suggestedName = randNameGen.getRandomName();
         String[] nameComponents = suggestedName.split(" ");
         String screenName = nameComponents[0] + nameComponents[1] + randNameGen.getRandomNameSuffix();
@@ -337,7 +351,7 @@
         twUserId++;
     }
 
-    public static void getCorrespondingTwitterUser(FacebookUser fbUser) {
+    public void getCorrespondingTwitterUser(FacebookUser fbUser) {
         String screenName = fbUser.getName().substring(0, fbUser.getName().lastIndexOf(commonUserFbSuffix))
                 + commonUserTwSuffix;
         String name = screenName.split(" ")[0];
@@ -414,16 +428,12 @@
         }
 
         public static void main(String args[]) throws Exception {
-            Date date = new Date(2, 20, 2012);
             RandomDateGenerator dgen = new RandomDateGenerator(new Date(1, 1, 2005), new Date(8, 20, 2012));
             while (true) {
                 Date nextDate = dgen.getNextRandomDate();
                 if (nextDate.getDay() == 0) {
                     throw new Exception("invalid date " + nextDate);
                 }
-
-                // System.out.println(" original date: " + date);
-                System.out.println(nextDate);
             }
         }
     }
@@ -730,7 +740,6 @@
     public static class FileUtil {
 
         public static List<String> listyFile(File file) throws IOException {
-
             BufferedReader reader = new BufferedReader(new FileReader(file));
             String line;
             List<String> list = new ArrayList<String>();
@@ -741,6 +750,7 @@
                 }
                 list.add(line);
             }
+            reader.close();
             return list;
         }
 
@@ -821,13 +831,12 @@
 
     public static class RandomLocationGenerator {
 
-        private Random random = new Random();
-
         private final int beginLat;
         private final int endLat;
         private final int beginLong;
         private final int endLong;
 
+        private Random random = new Random();
         private Point point;
 
         public RandomLocationGenerator(int beginLat, int endLat, int beginLong, int endLong) {
@@ -1266,7 +1275,7 @@
 
     public static class TweetMessage {
 
-        private String tweetid;
+        private long tweetid;
         private TwitterUser user;
         private Point senderLocation;
         private DateTime sendTime;
@@ -1277,7 +1286,7 @@
 
         }
 
-        public TweetMessage(String tweetid, TwitterUser user, Point senderLocation, DateTime sendTime,
+        public TweetMessage(long tweetid, TwitterUser user, Point senderLocation, DateTime sendTime,
                 List<String> referredTopics, Message messageText) {
             this.tweetid = tweetid;
             this.user = user;
@@ -1287,7 +1296,7 @@
             this.messageText = messageText;
         }
 
-        public void reset(String tweetid, TwitterUser user, Point senderLocation, DateTime sendTime,
+        public void reset(long tweetid, TwitterUser user, Point senderLocation, DateTime sendTime,
                 List<String> referredTopics, Message messageText) {
             this.tweetid = tweetid;
             this.user = user;
@@ -1301,7 +1310,7 @@
             StringBuilder builder = new StringBuilder();
             builder.append("{");
             builder.append("\"tweetid\":");
-            builder.append("\"" + tweetid + "\"");
+            builder.append("int64(\"" + tweetid + "\")");
             builder.append(",");
             builder.append("\"user\":");
             builder.append(user);
@@ -1333,11 +1342,11 @@
             return new String(builder);
         }
 
-        public String getTweetid() {
+        public long getTweetid() {
             return tweetid;
         }
 
-        public void setTweetid(String tweetid) {
+        public void setTweetid(long tweetid) {
             this.tweetid = tweetid;
         }
 
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/DataGenerator2.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/DataGenerator2.java
deleted file mode 100644
index 3fa7170..0000000
--- a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/DataGenerator2.java
+++ /dev/null
@@ -1,2487 +0,0 @@
-package edu.uci.ics.asterix.tools.external.data;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.nio.CharBuffer;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Random;
-import java.util.UUID;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-
-public class DataGenerator2 {
-
-    private RandomDateGenerator randDateGen;
-    private RandomNameGenerator randNameGen;
-    private RandomEmploymentGenerator randEmpGen;
-    private RandomMessageGenerator randMessageGen;
-    private RandomLocationGenerator randLocationGen;
-
-    private DistributionHandler fbDistHandler;
-    private DistributionHandler twDistHandler;
-
-    private int totalFbMessages;
-    private int numFbOnlyUsers;
-    private int totalTwMessages;
-    private int numTwOnlyUsers;
-
-    private int numCommonUsers;
-
-    private int fbUserId;
-    private int twUserId;
-
-    private int fbMessageId;
-    private int twMessageId;
-
-    private Random random = new Random();
-
-    private String commonUserFbSuffix = "_fb";
-    private String commonUserTwSuffix = "_tw";
-
-    private String outputDir;
-
-    private PartitionConfiguration partition;
-
-    private FacebookUser fbUser = new FacebookUser();
-    private TwitterUser twUser = new TwitterUser();
-
-    private FacebookMessage fbMessage = new FacebookMessage();
-    private TweetMessage twMessage = new TweetMessage();
-
-    private int duration;
-
-    private DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
-
-    public DataGenerator2(String[] args) throws Exception {
-        String controllerInstallDir = args[0];
-        String partitionConfXML = controllerInstallDir + "/output/partition-conf.xml";
-        String partitionName = args[1];
-        partition = XMLUtil.getPartitionConfiguration(partitionConfXML, partitionName);
-
-        // 1
-        randDateGen = new RandomDateGenerator(new Date(1, 1, 2005), new Date(8, 20, 2012));
-
-        String firstNameFile = controllerInstallDir + "/metadata/firstNames.txt";
-        String lastNameFile = controllerInstallDir + "/metadata/lastNames.txt";
-        String vendorFile = controllerInstallDir + "/metadata/vendors.txt";
-        String jargonFile = controllerInstallDir + "/metadata/jargon.txt";
-        String orgList = controllerInstallDir + "/metadata/org_list.txt";
-
-        randNameGen = new RandomNameGenerator(firstNameFile, lastNameFile);
-        randEmpGen = new RandomEmploymentGenerator(90, new Date(1, 1, 2000), new Date(8, 20, 2012), orgList);
-        randLocationGen = new RandomLocationGenerator(24, 49, 66, 98);
-        randMessageGen = new RandomMessageGenerator(vendorFile, jargonFile);
-
-        totalFbMessages = partition.getTargetPartition().getFbMessageIdMax()
-                - partition.getTargetPartition().getFbMessageIdMin() + 1;
-        numFbOnlyUsers = (partition.getTargetPartition().getFbUserKeyMax()
-                - partition.getTargetPartition().getFbUserKeyMin() + 1)
-                - partition.getTargetPartition().getCommonUsers();
-
-        totalTwMessages = partition.getTargetPartition().getTwMessageIdMax()
-                - partition.getTargetPartition().getTwMessageIdMin() + 1;
-        numTwOnlyUsers = (partition.getTargetPartition().getTwUserKeyMax()
-                - partition.getTargetPartition().getTwUserKeyMin() + 1)
-                - partition.getTargetPartition().getCommonUsers();
-
-        numCommonUsers = partition.getTargetPartition().getCommonUsers();
-        fbDistHandler = new DistributionHandler(totalFbMessages, 0.5, numFbOnlyUsers + numCommonUsers);
-        twDistHandler = new DistributionHandler(totalTwMessages, 0.5, numTwOnlyUsers + numCommonUsers);
-
-        fbUserId = partition.getTargetPartition().getFbUserKeyMin();
-        twUserId = partition.getTargetPartition().getTwUserKeyMin();
-
-        fbMessageId = partition.getTargetPartition().getFbMessageIdMin();
-        twMessageId = partition.getTargetPartition().getTwMessageIdMin();
-
-        outputDir = partition.getSourcePartition().getPath();
-    }
-
-    public DataGenerator2(InitializationInfo info) {
-        initialize(info);
-    }
-
-    private void generateFacebookOnlyUsers(int numFacebookUsers) throws IOException {
-        FileAppender appender = FileUtil.getFileAppender(outputDir + "/" + "fb_users.adm", true, true);
-        FileAppender messageAppender = FileUtil.getFileAppender(outputDir + "/" + "fb_message.adm", true, true);
-
-        for (int i = 0; i < numFacebookUsers; i++) {
-            getFacebookUser(null);
-            appender.appendToFile(fbUser.toString());
-            generateFacebookMessages(fbUser, messageAppender, -1);
-        }
-        appender.close();
-        messageAppender.close();
-    }
-
-    private void generateTwitterOnlyUsers(int numTwitterUsers) throws IOException {
-        FileAppender appender = FileUtil.getFileAppender(outputDir + "/" + "tw_users.adm", true, true);
-        FileAppender messageAppender = FileUtil.getFileAppender(outputDir + "/" + "tw_message.adm", true, true);
-
-        for (int i = 0; i < numTwitterUsers; i++) {
-            getTwitterUser(null);
-            appender.appendToFile(twUser.toString());
-            generateTwitterMessages(twUser, messageAppender, -1);
-        }
-        appender.close();
-        messageAppender.close();
-    }
-
-    private void generateCommonUsers() throws IOException {
-        FileAppender fbAppender = FileUtil.getFileAppender(outputDir + "/" + "fb_users.adm", true, false);
-        FileAppender twAppender = FileUtil.getFileAppender(outputDir + "/" + "tw_users.adm", true, false);
-        FileAppender fbMessageAppender = FileUtil.getFileAppender(outputDir + "/" + "fb_message.adm", true, false);
-        FileAppender twMessageAppender = FileUtil.getFileAppender(outputDir + "/" + "tw_message.adm", true, false);
-
-        for (int i = 0; i < numCommonUsers; i++) {
-            getFacebookUser(commonUserFbSuffix);
-            fbAppender.appendToFile(fbUser.toString());
-            generateFacebookMessages(fbUser, fbMessageAppender, -1);
-
-            getCorrespondingTwitterUser(fbUser);
-            twAppender.appendToFile(twUser.toString());
-            generateTwitterMessages(twUser, twMessageAppender, -1);
-        }
-
-        fbAppender.close();
-        twAppender.close();
-        fbMessageAppender.close();
-        twMessageAppender.close();
-    }
-
-    private void generateFacebookMessages(FacebookUser user, FileAppender appender, int numMsg) throws IOException {
-        Message message;
-        int numMessages = 0;
-        if (numMsg == -1) {
-            numMessages = fbDistHandler
-                    .getFromDistribution(fbUserId - partition.getTargetPartition().getFbUserKeyMin());
-        }
-        for (int i = 0; i < numMessages; i++) {
-            message = randMessageGen.getNextRandomMessage();
-            Point location = randLocationGen.getRandomPoint();
-            fbMessage.reset(fbMessageId++, user.getId(), random.nextInt(totalFbMessages + 1), location, message);
-            appender.appendToFile(fbMessage.toString());
-        }
-    }
-
-    private void generateTwitterMessages(TwitterUser user, FileAppender appender, int numMsg) throws IOException {
-        Message message;
-        int numMessages = 0;
-        if (numMsg == -1) {
-            numMessages = twDistHandler
-                    .getFromDistribution(twUserId - partition.getTargetPartition().getTwUserKeyMin());
-        }
-
-        for (int i = 0; i < numMessages; i++) {
-            message = randMessageGen.getNextRandomMessage();
-            Point location = randLocationGen.getRandomPoint();
-            DateTime sendTime = randDateGen.getNextRandomDatetime();
-            twMessage.reset(twMessageId, user, location, sendTime, message.getReferredTopics(), message);
-            twMessageId++;
-            appender.appendToFile(twMessage.toString());
-        }
-    }
-
-    public Iterator<TweetMessage> getTwitterMessageIterator(int partition, byte seed) {
-        return new TweetMessageIterator(duration, partition, seed);
-    }
-
-    public class TweetMessageIterator implements Iterator<TweetMessage> {
-
-        private final int duration;
-        private long startTime = 0;
-        private int partition;
-        private final GULongIDGenerator idGen;
-        
-        public TweetMessageIterator(int duration,int partition, byte seed) {
-            this.duration = duration;
-            this.partition = partition;
-            this.idGen = new GULongIDGenerator(partition, seed);
-        }
-
-        @Override
-        public boolean hasNext() {
-            if (startTime == 0) {
-                startTime = System.currentTimeMillis();
-            }
-            return System.currentTimeMillis() - startTime < duration * 1000;
-        }
-
-        @Override
-        public TweetMessage next() {
-            getTwitterUser(null);
-            Message message = randMessageGen.getNextRandomMessage();
-            Point location = randLocationGen.getRandomPoint();
-            DateTime sendTime = randDateGen.getNextRandomDatetime();
-            twMessage.reset(idGen.getNextULong(), twUser, location, sendTime, message.getReferredTopics(),
-                    message);
-            twMessageId++;
-            if (twUserId > numTwOnlyUsers) {
-                twUserId = 1;
-            }
-            return twMessage;
-
-        }
-
-        @Override
-        public void remove() {
-            // TODO Auto-generated method stub
-
-        }
-
-    }
-
-    public static class InitializationInfo {
-        public Date startDate = new Date(1, 1, 2005);
-        public Date endDate = new Date(8, 20, 2012);
-        public String[] lastNames = DataGenerator.lastNames;
-        public String[] firstNames = DataGenerator.firstNames;
-        public String[] vendors = DataGenerator.vendors;
-        public String[] jargon = DataGenerator.jargon;
-        public String[] org_list = DataGenerator.org_list;
-        public int percentEmployed = 90;
-        public Date employmentStartDate = new Date(1, 1, 2000);
-        public Date employmentEndDate = new Date(31, 12, 2012);
-        public int totalFbMessages;
-        public int numFbOnlyUsers;
-        public int totalTwMessages;
-        public int numTwOnlyUsers = 5000;
-        public int numCommonUsers;
-        public int fbUserIdMin;
-        public int fbMessageIdMin;
-        public int twUserIdMin;
-        public int twMessageIdMin;
-        public int timeDurationInSecs = 60;
-
-    }
-
-    public void initialize(InitializationInfo info) {
-        randDateGen = new RandomDateGenerator(info.startDate, info.endDate);
-        randNameGen = new RandomNameGenerator(info.firstNames, info.lastNames);
-        randEmpGen = new RandomEmploymentGenerator(info.percentEmployed, info.employmentStartDate,
-                info.employmentEndDate, info.org_list);
-        randLocationGen = new RandomLocationGenerator(24, 49, 66, 98);
-        randMessageGen = new RandomMessageGenerator(info.vendors, info.jargon);
-        fbDistHandler = new DistributionHandler(info.totalFbMessages, 0.5, info.numFbOnlyUsers + info.numCommonUsers);
-        twDistHandler = new DistributionHandler(info.totalTwMessages, 0.5, info.numTwOnlyUsers + info.numCommonUsers);
-        fbUserId = info.fbUserIdMin;
-        twUserId = info.twUserIdMin;
-
-        fbMessageId = info.fbMessageIdMin;
-        twMessageId = info.fbMessageIdMin;
-        duration = info.timeDurationInSecs;
-    }
-
-    public static void main(String args[]) throws Exception {
-
-        String controllerInstallDir = null;
-        if (args.length < 2) {
-            printUsage();
-            System.exit(1);
-        }
-
-        DataGenerator2 dataGenerator = new DataGenerator2(args);
-        dataGenerator.generateData();
-    }
-
-    public static void printUsage() {
-        System.out.println(" Error: Invalid number of arguments ");
-        System.out.println(" Usage :" + " DataGenerator <path to configuration file> <partition name> ");
-    }
-
-    public void generateData() throws IOException {
-        generateFacebookOnlyUsers(numFbOnlyUsers);
-        generateTwitterOnlyUsers(numTwOnlyUsers);
-        generateCommonUsers();
-        System.out.println("Partition :" + partition.getTargetPartition().getName() + " finished");
-    }
-
-    public void getFacebookUser(String usernameSuffix) {
-        String suggestedName = randNameGen.getRandomName();
-        String[] nameComponents = suggestedName.split(" ");
-        String name = nameComponents[0] + nameComponents[1];
-        if (usernameSuffix != null) {
-            name = name + usernameSuffix;
-        }
-        String alias = nameComponents[0];
-        String userSince = randDateGen.getNextRandomDatetime().toString();
-        int numFriends = random.nextInt(25);
-        int[] friendIds = RandomUtil.getKFromN(numFriends, (numFbOnlyUsers + numCommonUsers));
-        Employment emp = randEmpGen.getRandomEmployment();
-        fbUser.reset(fbUserId++, alias, name, userSince, friendIds, emp);
-    }
-
-    public void getTwitterUser(String usernameSuffix) {
-        String suggestedName = randNameGen.getRandomName();
-        String[] nameComponents = suggestedName.split(" ");
-        String screenName = nameComponents[0] + nameComponents[1] + randNameGen.getRandomNameSuffix();
-        String name = suggestedName;
-        if (usernameSuffix != null) {
-            name = name + usernameSuffix;
-        }
-        int numFriends = random.nextInt((int) (100)); // draw from Zipfian
-        int statusesCount = random.nextInt(500); // draw from Zipfian
-        int followersCount = random.nextInt((int) (200));
-        twUser.reset(screenName, numFriends, statusesCount, name, followersCount);
-        twUserId++;
-    }
-
-    public void getCorrespondingTwitterUser(FacebookUser fbUser) {
-        String screenName = fbUser.getName().substring(0, fbUser.getName().lastIndexOf(commonUserFbSuffix))
-                + commonUserTwSuffix;
-        String name = screenName.split(" ")[0];
-        int numFriends = random.nextInt((int) ((numTwOnlyUsers + numCommonUsers)));
-        int statusesCount = random.nextInt(500); // draw from Zipfian
-        int followersCount = random.nextInt((int) (numTwOnlyUsers + numCommonUsers));
-        twUser.reset(screenName, numFriends, statusesCount, name, followersCount);
-    }
-
-    public static class RandomDateGenerator {
-
-        private final Date startDate;
-        private final Date endDate;
-        private final Random random = new Random();
-        private final int yearDifference;
-        private Date workingDate;
-        private Date recentDate;
-        private DateTime dateTime;
-
-        public RandomDateGenerator(Date startDate, Date endDate) {
-            this.startDate = startDate;
-            this.endDate = endDate;
-            yearDifference = endDate.getYear() - startDate.getYear() + 1;
-            workingDate = new Date();
-            recentDate = new Date();
-            dateTime = new DateTime();
-        }
-
-        public Date getStartDate() {
-            return startDate;
-        }
-
-        public Date getEndDate() {
-            return endDate;
-        }
-
-        public Date getNextRandomDate() {
-            int year = random.nextInt(yearDifference) + startDate.getYear();
-            int month;
-            int day;
-            if (year == endDate.getYear()) {
-                month = random.nextInt(endDate.getMonth()) + 1;
-                if (month == endDate.getMonth()) {
-                    day = random.nextInt(endDate.getDay()) + 1;
-                } else {
-                    day = random.nextInt(28) + 1;
-                }
-            } else {
-                month = random.nextInt(12) + 1;
-                day = random.nextInt(28) + 1;
-            }
-            workingDate.reset(month, day, year);
-            return workingDate;
-        }
-
-        public DateTime getNextRandomDatetime() {
-            Date randomDate = getNextRandomDate();
-            dateTime.reset(randomDate);
-            return dateTime;
-        }
-
-        public Date getNextRecentDate(Date date) {
-            int year = date.getYear()
-                    + (date.getYear() == endDate.getYear() ? 0 : random.nextInt(endDate.getYear() - date.getYear()));
-            int month = (year == endDate.getYear()) ? date.getMonth() == endDate.getMonth() ? (endDate.getMonth())
-                    : (date.getMonth() + random.nextInt(endDate.getMonth() - date.getMonth())) : random.nextInt(12) + 1;
-
-            int day = (year == endDate.getYear()) ? month == endDate.getMonth() ? date.getDay() == endDate.getDay() ? endDate
-                    .getDay() : date.getDay() + random.nextInt(endDate.getDay() - date.getDay())
-                    : random.nextInt(28) + 1
-                    : random.nextInt(28) + 1;
-            recentDate.reset(month, day, year);
-            return recentDate;
-        }
-
-        public static void main(String args[]) throws Exception {
-            Date date = new Date(2, 20, 2012);
-            RandomDateGenerator dgen = new RandomDateGenerator(new Date(1, 1, 2005), new Date(8, 20, 2012));
-            while (true) {
-                Date nextDate = dgen.getNextRandomDate();
-                if (nextDate.getDay() == 0) {
-                    throw new Exception("invalid date " + nextDate);
-                }
-
-                // System.out.println(" original date: " + date);
-                System.out.println(nextDate);
-            }
-        }
-    }
-
-    public static class DateTime extends Date {
-
-        private String hour = "10";
-        private String min = "10";
-        private String sec = "00";
-        private long chrononTime;
-
-        public DateTime(int month, int day, int year, String hour, String min, String sec) {
-            super(month, day, year);
-            this.hour = hour;
-            this.min = min;
-            this.sec = sec;
-            chrononTime = new java.util.Date(year, month, day, Integer.parseInt(hour), Integer.parseInt(min),
-                    Integer.parseInt(sec)).getTime();
-        }
-
-        public void reset(int month, int day, int year, String hour, String min, String sec) {
-            super.setDay(month);
-            super.setDay(day);
-            super.setYear(year);
-            this.hour = hour;
-            this.min = min;
-            this.sec = sec;
-            chrononTime = new java.util.Date(year, month, day, Integer.parseInt(hour), Integer.parseInt(min),
-                    Integer.parseInt(sec)).getTime();
-        }
-
-        public DateTime() {
-        }
-
-        public DateTime(Date date) {
-            super(date.getMonth(), date.getDay(), date.getYear());
-        }
-
-        public void reset(Date date) {
-            reset(date.getMonth(), date.getDay(), date.getYear());
-        }
-
-        public DateTime(Date date, int hour, int min, int sec) {
-            super(date.getMonth(), date.getDay(), date.getYear());
-            this.hour = (hour < 10) ? "0" : "" + hour;
-            this.min = (min < 10) ? "0" : "" + min;
-            this.sec = (sec < 10) ? "0" : "" + sec;
-        }
-
-        public long getChrononTime() {
-            return chrononTime;
-        }
-
-        public String getHour() {
-            return hour;
-        }
-
-        public String getMin() {
-            return min;
-        }
-
-        public String getSec() {
-            return sec;
-        }
-
-        public String toString() {
-            StringBuilder builder = new StringBuilder();
-            builder.append("datetime");
-            builder.append("(\"");
-            builder.append(super.getYear());
-            builder.append("-");
-            builder.append(super.getMonth() < 10 ? "0" + super.getMonth() : super.getMonth());
-            builder.append("-");
-            builder.append(super.getDay() < 10 ? "0" + super.getDay() : super.getDay());
-            builder.append("T");
-            builder.append(hour + ":" + min + ":" + sec);
-            builder.append("\")");
-            return builder.toString();
-        }
-    }
-
-    public static class Message {
-
-        private char[] message = new char[500];
-        private List<String> referredTopics;
-        private int length;
-
-        public Message(char[] m, List<String> referredTopics) {
-            System.arraycopy(m, 0, message, 0, m.length);
-            length = m.length;
-            this.referredTopics = referredTopics;
-        }
-
-        public Message() {
-            referredTopics = new ArrayList<String>();
-            length = 0;
-        }
-
-        public char[] getMessage() {
-            return message;
-        }
-
-        public List<String> getReferredTopics() {
-            return referredTopics;
-        }
-
-        public void reset(char[] m, int offset, int length, List<String> referredTopics) {
-            System.arraycopy(m, offset, message, 0, length);
-            this.length = length;
-            this.referredTopics = referredTopics;
-        }
-
-        public int getLength() {
-            return length;
-        }
-
-        public char charAt(int index) {
-            return message[index];
-        }
-
-    }
-
-    public static class Point {
-
-        private float latitude;
-        private float longitude;
-
-        public float getLatitude() {
-            return latitude;
-        }
-
-        public float getLongitude() {
-            return longitude;
-        }
-
-        public Point(float latitude, float longitude) {
-            this.latitude = latitude;
-            this.longitude = longitude;
-        }
-
-        public void reset(float latitude, float longitude) {
-            this.latitude = latitude;
-            this.longitude = longitude;
-        }
-
-        public Point() {
-        }
-
-        public String toString() {
-            StringBuilder builder = new StringBuilder();
-            builder.append("point(\"" + latitude + "," + longitude + "\")");
-            return builder.toString();
-        }
-    }
-
-    public static class RandomNameGenerator {
-
-        private String[] firstNames;
-        private String[] lastNames;
-
-        private final Random random = new Random();
-
-        private final String[] connectors = new String[] { "_", "#", "$", "@" };
-
-        public RandomNameGenerator(String firstNameFilePath, String lastNameFilePath) throws IOException {
-            firstNames = FileUtil.listyFile(new File(firstNameFilePath)).toArray(new String[] {});
-            lastNames = FileUtil.listyFile(new File(lastNameFilePath)).toArray(new String[] {});
-        }
-
-        public RandomNameGenerator(String[] firstNames, String[] lastNames) {
-            this.firstNames = firstNames;
-            this.lastNames = lastNames;
-        }
-
-        public String getRandomName() {
-            String name;
-            name = getSuggestedName();
-            return name;
-
-        }
-
-        private String getSuggestedName() {
-            int firstNameIndex = random.nextInt(firstNames.length);
-            int lastNameIndex = random.nextInt(lastNames.length);
-            String suggestedName = firstNames[firstNameIndex] + " " + lastNames[lastNameIndex];
-            return suggestedName;
-        }
-
-        public String getRandomNameSuffix() {
-            return connectors[random.nextInt(connectors.length)] + random.nextInt(1000);
-        }
-    }
-
-    public static class RandomMessageGenerator {
-
-        private final MessageTemplate messageTemplate;
-
-        public RandomMessageGenerator(String vendorFilePath, String jargonFilePath) throws IOException {
-            List<String> vendors = FileUtil.listyFile(new File(vendorFilePath));
-            List<String> jargon = FileUtil.listyFile(new File(jargonFilePath));
-            this.messageTemplate = new MessageTemplate(vendors, jargon);
-        }
-
-        public RandomMessageGenerator(String[] vendors, String[] jargon) {
-            List<String> vendorList = new ArrayList<String>();
-            for (String v : vendors) {
-                vendorList.add(v);
-            }
-            List<String> jargonList = new ArrayList<String>();
-            for (String j : jargon) {
-                jargonList.add(j);
-            }
-            this.messageTemplate = new MessageTemplate(vendorList, jargonList);
-        }
-
-        public Message getNextRandomMessage() {
-            return messageTemplate.getNextMessage();
-        }
-    }
-
-    public static class AbstractMessageTemplate {
-
-        protected final Random random = new Random();
-
-        protected String[] positiveVerbs = new String[] { "like", "love" };
-        protected String[] negativeVerbs = new String[] { "dislike", "hate", "can't stand" };
-
-        protected String[] negativeAdjectives = new String[] { "horrible", "bad", "terrible", "OMG" };
-        protected String[] postiveAdjectives = new String[] { "good", "awesome", "amazing", "mind-blowing" };
-
-        protected String[] otherWords = new String[] { "the", "its" };
-    }
-
-    public static class MessageTemplate extends AbstractMessageTemplate {
-
-        private List<String> vendors;
-        private List<String> jargon;
-        private CharBuffer buffer;
-        private List<String> referredTopics;
-        private Message message = new Message();
-
-        public MessageTemplate(List<String> vendors, List<String> jargon) {
-            this.vendors = vendors;
-            this.jargon = jargon;
-            buffer = CharBuffer.allocate(2500);
-            referredTopics = new ArrayList<String>();
-        }
-
-        public Message getNextMessage() {
-            buffer.position(0);
-            buffer.limit(2500);
-            referredTopics.clear();
-            boolean isPositive = random.nextBoolean();
-            String[] verbArray = isPositive ? positiveVerbs : negativeVerbs;
-            String[] adjectiveArray = isPositive ? postiveAdjectives : negativeAdjectives;
-            String verb = verbArray[random.nextInt(verbArray.length)];
-            String adjective = adjectiveArray[random.nextInt(adjectiveArray.length)];
-
-            buffer.put(" ");
-            buffer.put(verb);
-            buffer.put(" ");
-            String vendor = vendors.get(random.nextInt(vendors.size()));
-            referredTopics.add(vendor);
-            buffer.append(vendor);
-            buffer.append(" ");
-            buffer.append(otherWords[random.nextInt(otherWords.length)]);
-            buffer.append(" ");
-            String jargonTerm = jargon.get(random.nextInt(jargon.size()));
-            referredTopics.add(jargonTerm);
-            buffer.append(jargonTerm);
-            buffer.append(" is ");
-            buffer.append(adjective);
-            if (random.nextBoolean()) {
-                buffer.append(isPositive ? ":)" : ":(");
-            }
-
-            buffer.flip();
-            message.reset(buffer.array(), 0, buffer.limit(), referredTopics);
-            return message;
-        }
-    }
-
-    public static class RandomUtil {
-
-        public static Random random = new Random();
-
-        public static int[] getKFromN(int k, int n) {
-            int[] result = new int[k];
-            int cnt = 0;
-            HashSet<Integer> values = new HashSet<Integer>();
-            while (cnt < k) {
-                int val = random.nextInt(n + 1);
-                if (values.contains(val)) {
-                    continue;
-                }
-
-                result[cnt++] = val;
-                values.add(val);
-            }
-            return result;
-        }
-    }
-
-    public static class FileUtil {
-
-        public static List<String> listyFile(File file) throws IOException {
-
-            BufferedReader reader = new BufferedReader(new FileReader(file));
-            String line;
-            List<String> list = new ArrayList<String>();
-            while (true) {
-                line = reader.readLine();
-                if (line == null) {
-                    break;
-                }
-                list.add(line);
-            }
-            return list;
-        }
-
-        public static FileAppender getFileAppender(String filePath, boolean createIfNotExists, boolean overwrite)
-                throws IOException {
-            return new FileAppender(filePath, createIfNotExists, overwrite);
-        }
-    }
-
-    public static class FileAppender {
-
-        private final BufferedWriter writer;
-
-        public FileAppender(String filePath, boolean createIfNotExists, boolean overwrite) throws IOException {
-            File file = new File(filePath);
-            if (!file.exists()) {
-                if (createIfNotExists) {
-                    new File(file.getParent()).mkdirs();
-                } else {
-                    throw new IOException("path " + filePath + " does not exists");
-                }
-            }
-            this.writer = new BufferedWriter(new FileWriter(file, !overwrite));
-        }
-
-        public void appendToFile(String content) throws IOException {
-            writer.append(content);
-            writer.append("\n");
-        }
-
-        public void close() throws IOException {
-            writer.close();
-        }
-    }
-
-    public static class RandomEmploymentGenerator {
-
-        private final int percentEmployed;
-        private final Random random = new Random();
-        private final RandomDateGenerator randDateGen;
-        private final List<String> organizations;
-        private Employment emp;
-
-        public RandomEmploymentGenerator(int percentEmployed, Date beginEmpDate, Date endEmpDate, String orgListPath)
-                throws IOException {
-            this.percentEmployed = percentEmployed;
-            this.randDateGen = new RandomDateGenerator(beginEmpDate, endEmpDate);
-            organizations = FileUtil.listyFile(new File(orgListPath));
-            emp = new Employment();
-        }
-
-        public RandomEmploymentGenerator(int percentEmployed, Date beginEmpDate, Date endEmpDate, String[] orgList) {
-            this.percentEmployed = percentEmployed;
-            this.randDateGen = new RandomDateGenerator(beginEmpDate, endEmpDate);
-            organizations = new ArrayList<String>();
-            for (String org : orgList) {
-                organizations.add(org);
-            }
-            emp = new Employment();
-        }
-
-        public Employment getRandomEmployment() {
-            int empployed = random.nextInt(100) + 1;
-            boolean isEmployed = false;
-            if (empployed <= percentEmployed) {
-                isEmployed = true;
-            }
-            Date startDate = randDateGen.getNextRandomDate();
-            Date endDate = null;
-            if (!isEmployed) {
-                endDate = randDateGen.getNextRecentDate(startDate);
-            }
-            String org = organizations.get(random.nextInt(organizations.size()));
-            emp.reset(org, startDate, endDate);
-            return emp;
-        }
-    }
-
-    public static class RandomLocationGenerator {
-
-        private Random random = new Random();
-
-        private final int beginLat;
-        private final int endLat;
-        private final int beginLong;
-        private final int endLong;
-
-        private Point point;
-
-        public RandomLocationGenerator(int beginLat, int endLat, int beginLong, int endLong) {
-            this.beginLat = beginLat;
-            this.endLat = endLat;
-            this.beginLong = beginLong;
-            this.endLong = endLong;
-            this.point = new Point();
-        }
-
-        public Point getRandomPoint() {
-            int latMajor = beginLat + random.nextInt(endLat - beginLat);
-            int latMinor = random.nextInt(100);
-            float latitude = latMajor + ((float) latMinor) / 100;
-
-            int longMajor = beginLong + random.nextInt(endLong - beginLong);
-            int longMinor = random.nextInt(100);
-            float longitude = longMajor + ((float) longMinor) / 100;
-
-            point.reset(latitude, longitude);
-            return point;
-        }
-
-    }
-
-    public static class PartitionConfiguration {
-
-        private final TargetPartition targetPartition;
-        private final SourcePartition sourcePartition;
-
-        public PartitionConfiguration(SourcePartition sourcePartition, TargetPartition targetPartition) {
-            this.sourcePartition = sourcePartition;
-            this.targetPartition = targetPartition;
-        }
-
-        public TargetPartition getTargetPartition() {
-            return targetPartition;
-        }
-
-        public SourcePartition getSourcePartition() {
-            return sourcePartition;
-        }
-
-    }
-
-    public static class SourcePartition {
-
-        private final String name;
-        private final String host;
-        private final String path;
-
-        public SourcePartition(String name, String host, String path) {
-            this.name = name;
-            this.host = host;
-            this.path = path;
-        }
-
-        public String getName() {
-            return name;
-        }
-
-        public String getHost() {
-            return host;
-        }
-
-        public String getPath() {
-            return path;
-        }
-    }
-
-    public static class TargetPartition {
-        private final String name;
-        private final String host;
-        private final String path;
-        private final int fbUserKeyMin;
-        private final int fbUserKeyMax;
-        private final int twUserKeyMin;
-        private final int twUserKeyMax;
-        private final int fbMessageIdMin;
-        private final int fbMessageIdMax;
-        private final int twMessageIdMin;
-        private final int twMessageIdMax;
-        private final int commonUsers;
-
-        public TargetPartition(String partitionName, String host, String path, int fbUserKeyMin, int fbUserKeyMax,
-                int twUserKeyMin, int twUserKeyMax, int fbMessageIdMin, int fbMessageIdMax, int twMessageIdMin,
-                int twMessageIdMax, int commonUsers) {
-            this.name = partitionName;
-            this.host = host;
-            this.path = path;
-            this.fbUserKeyMin = fbUserKeyMin;
-            this.fbUserKeyMax = fbUserKeyMax;
-            this.twUserKeyMin = twUserKeyMin;
-            this.twUserKeyMax = twUserKeyMax;
-            this.twMessageIdMin = twMessageIdMin;
-            this.twMessageIdMax = twMessageIdMax;
-            this.fbMessageIdMin = fbMessageIdMin;
-            this.fbMessageIdMax = fbMessageIdMax;
-            this.commonUsers = commonUsers;
-        }
-
-        public String toString() {
-            StringBuilder builder = new StringBuilder();
-            builder.append(name);
-            builder.append(" ");
-            builder.append(host);
-            builder.append("\n");
-            builder.append(path);
-            builder.append("\n");
-            builder.append("fbUser:key:min");
-            builder.append(fbUserKeyMin);
-
-            builder.append("\n");
-            builder.append("fbUser:key:max");
-            builder.append(fbUserKeyMax);
-
-            builder.append("\n");
-            builder.append("twUser:key:min");
-            builder.append(twUserKeyMin);
-
-            builder.append("\n");
-            builder.append("twUser:key:max");
-            builder.append(twUserKeyMax);
-
-            builder.append("\n");
-            builder.append("fbMessage:key:min");
-            builder.append(fbMessageIdMin);
-
-            builder.append("\n");
-            builder.append("fbMessage:key:max");
-            builder.append(fbMessageIdMax);
-
-            builder.append("\n");
-            builder.append("twMessage:key:min");
-            builder.append(twMessageIdMin);
-
-            builder.append("\n");
-            builder.append("twMessage:key:max");
-            builder.append(twMessageIdMax);
-
-            builder.append("\n");
-            builder.append("twMessage:key:max");
-            builder.append(twMessageIdMax);
-
-            builder.append("\n");
-            builder.append("commonUsers");
-            builder.append(commonUsers);
-
-            return new String(builder);
-        }
-
-        public String getName() {
-            return name;
-        }
-
-        public String getHost() {
-            return host;
-        }
-
-        public int getFbUserKeyMin() {
-            return fbUserKeyMin;
-        }
-
-        public int getFbUserKeyMax() {
-            return fbUserKeyMax;
-        }
-
-        public int getTwUserKeyMin() {
-            return twUserKeyMin;
-        }
-
-        public int getTwUserKeyMax() {
-            return twUserKeyMax;
-        }
-
-        public int getFbMessageIdMin() {
-            return fbMessageIdMin;
-        }
-
-        public int getFbMessageIdMax() {
-            return fbMessageIdMax;
-        }
-
-        public int getTwMessageIdMin() {
-            return twMessageIdMin;
-        }
-
-        public int getTwMessageIdMax() {
-            return twMessageIdMax;
-        }
-
-        public int getCommonUsers() {
-            return commonUsers;
-        }
-
-        public String getPath() {
-            return path;
-        }
-    }
-
-    public static class Employment {
-
-        private String organization;
-        private Date startDate;
-        private Date endDate;
-
-        public Employment(String organization, Date startDate, Date endDate) {
-            this.organization = organization;
-            this.startDate = startDate;
-            this.endDate = endDate;
-        }
-
-        public Employment() {
-        }
-
-        public void reset(String organization, Date startDate, Date endDate) {
-            this.organization = organization;
-            this.startDate = startDate;
-            this.endDate = endDate;
-        }
-
-        public String getOrganization() {
-            return organization;
-        }
-
-        public Date getStartDate() {
-            return startDate;
-        }
-
-        public Date getEndDate() {
-            return endDate;
-        }
-
-        public String toString() {
-            StringBuilder builder = new StringBuilder("");
-            builder.append("{");
-            builder.append("\"organization-name\":");
-            builder.append("\"" + organization + "\"");
-            builder.append(",");
-            builder.append("\"start-date\":");
-            builder.append(startDate);
-            if (endDate != null) {
-                builder.append(",");
-                builder.append("\"end-date\":");
-                builder.append(endDate);
-            }
-            builder.append("}");
-            return new String(builder);
-        }
-
-    }
-
-    public static class FacebookMessage {
-
-        private int messageId;
-        private int authorId;
-        private int inResponseTo;
-        private Point senderLocation;
-        private Message message;
-
-        public int getMessageId() {
-            return messageId;
-        }
-
-        public int getAuthorID() {
-            return authorId;
-        }
-
-        public Point getSenderLocation() {
-            return senderLocation;
-        }
-
-        public Message getMessage() {
-            return message;
-        }
-
-        public int getInResponseTo() {
-            return inResponseTo;
-        }
-
-        public FacebookMessage() {
-
-        }
-
-        public FacebookMessage(int messageId, int authorId, int inResponseTo, Point senderLocation, Message message) {
-            this.messageId = messageId;
-            this.authorId = authorId;
-            this.inResponseTo = inResponseTo;
-            this.senderLocation = senderLocation;
-            this.message = message;
-        }
-
-        public void reset(int messageId, int authorId, int inResponseTo, Point senderLocation, Message message) {
-            this.messageId = messageId;
-            this.authorId = authorId;
-            this.inResponseTo = inResponseTo;
-            this.senderLocation = senderLocation;
-            this.message = message;
-        }
-
-        public String toString() {
-            StringBuilder builder = new StringBuilder();
-            builder.append("{");
-            builder.append("\"message-id\":");
-            builder.append(messageId);
-            builder.append(",");
-            builder.append("\"author-id\":");
-            builder.append(authorId);
-            builder.append(",");
-            builder.append("\"in-response-to\":");
-            builder.append(inResponseTo);
-            builder.append(",");
-            builder.append("\"sender-location\":");
-            builder.append(senderLocation);
-            builder.append(",");
-            builder.append("\"message\":");
-            builder.append("\"");
-            for (int i = 0; i < message.getLength(); i++) {
-                builder.append(message.charAt(i));
-            }
-            builder.append("\"");
-            builder.append("}");
-            return new String(builder);
-        }
-    }
-
-    public static class FacebookUser {
-
-        private int id;
-        private String alias;
-        private String name;
-        private String userSince;
-        private int[] friendIds;
-        private Employment employment;
-
-        public FacebookUser() {
-
-        }
-
-        public FacebookUser(int id, String alias, String name, String userSince, int[] friendIds, Employment employment) {
-            this.id = id;
-            this.alias = alias;
-            this.name = name;
-            this.userSince = userSince;
-            this.friendIds = friendIds;
-            this.employment = employment;
-        }
-
-        public int getId() {
-            return id;
-        }
-
-        public String getAlias() {
-            return alias;
-        }
-
-        public String getName() {
-            return name;
-        }
-
-        public String getUserSince() {
-            return userSince;
-        }
-
-        public int[] getFriendIds() {
-            return friendIds;
-        }
-
-        public Employment getEmployment() {
-            return employment;
-        }
-
-        public String toString() {
-            StringBuilder builder = new StringBuilder();
-            builder.append("{");
-            builder.append("\"id\":" + id);
-            builder.append(",");
-            builder.append("\"alias\":" + "\"" + alias + "\"");
-            builder.append(",");
-            builder.append("\"name\":" + "\"" + name + "\"");
-            builder.append(",");
-            builder.append("\"user-since\":" + userSince);
-            builder.append(",");
-            builder.append("\"friend-ids\":");
-            builder.append("{{");
-            for (int i = 0; i < friendIds.length; i++) {
-                builder.append(friendIds[i]);
-                builder.append(",");
-            }
-            if (friendIds.length > 0) {
-                builder.deleteCharAt(builder.lastIndexOf(","));
-            }
-            builder.append("}}");
-            builder.append(",");
-            builder.append("\"employment\":");
-            builder.append("[");
-            builder.append(employment.toString());
-            builder.append("]");
-            builder.append("}");
-            return builder.toString();
-        }
-
-        public void setId(int id) {
-            this.id = id;
-        }
-
-        public void setAlias(String alias) {
-            this.alias = alias;
-        }
-
-        public void setName(String name) {
-            this.name = name;
-        }
-
-        public void setUserSince(String userSince) {
-            this.userSince = userSince;
-        }
-
-        public void setFriendIds(int[] friendIds) {
-            this.friendIds = friendIds;
-        }
-
-        public void setEmployment(Employment employment) {
-            this.employment = employment;
-        }
-
-        public void reset(int id, String alias, String name, String userSince, int[] friendIds, Employment employment) {
-            this.id = id;
-            this.alias = alias;
-            this.name = name;
-            this.userSince = userSince;
-            this.friendIds = friendIds;
-            this.employment = employment;
-        }
-    }
-
-    public static class TweetMessage {
-
-        private long tweetid;
-        private TwitterUser user;
-        private Point senderLocation;
-        private DateTime sendTime;
-        private List<String> referredTopics;
-        private Message messageText;
-
-        public TweetMessage() {
-
-        }
-
-        public TweetMessage(long tweetid, TwitterUser user, Point senderLocation, DateTime sendTime,
-                List<String> referredTopics, Message messageText) {
-            this.tweetid = tweetid;
-            this.user = user;
-            this.senderLocation = senderLocation;
-            this.sendTime = sendTime;
-            this.referredTopics = referredTopics;
-            this.messageText = messageText;
-        }
-
-        public void reset(long tweetid, TwitterUser user, Point senderLocation, DateTime sendTime,
-                List<String> referredTopics, Message messageText) {
-            this.tweetid = tweetid;
-            this.user = user;
-            this.senderLocation = senderLocation;
-            this.sendTime = sendTime;
-            this.referredTopics = referredTopics;
-            this.messageText = messageText;
-        }
-
-        public String toString() {
-            StringBuilder builder = new StringBuilder();
-            builder.append("{");
-            builder.append("\"tweetid\":");
-            builder.append("int64(\"" + tweetid + "\")");
-            builder.append(",");
-            builder.append("\"user\":");
-            builder.append(user);
-            builder.append(",");
-            builder.append("\"sender-location\":");
-            builder.append(senderLocation);
-            builder.append(",");
-            builder.append("\"send-time\":");
-            builder.append(sendTime);
-            builder.append(",");
-            builder.append("\"referred-topics\":");
-            builder.append("{{");
-            for (String topic : referredTopics) {
-                builder.append("\"" + topic + "\"");
-                builder.append(",");
-            }
-            if (referredTopics.size() > 0) {
-                builder.deleteCharAt(builder.lastIndexOf(","));
-            }
-            builder.append("}}");
-            builder.append(",");
-            builder.append("\"message-text\":");
-            builder.append("\"");
-            for (int i = 0; i < messageText.getLength(); i++) {
-                builder.append(messageText.charAt(i));
-            }
-            builder.append("\"");
-            builder.append("}");
-            return new String(builder);
-        }
-
-        public long getTweetid() {
-            return tweetid;
-        }
-
-        public void setTweetid(long tweetid) {
-            this.tweetid = tweetid;
-        }
-
-        public TwitterUser getUser() {
-            return user;
-        }
-
-        public void setUser(TwitterUser user) {
-            this.user = user;
-        }
-
-        public Point getSenderLocation() {
-            return senderLocation;
-        }
-
-        public void setSenderLocation(Point senderLocation) {
-            this.senderLocation = senderLocation;
-        }
-
-        public DateTime getSendTime() {
-            return sendTime;
-        }
-
-        public void setSendTime(DateTime sendTime) {
-            this.sendTime = sendTime;
-        }
-
-        public List<String> getReferredTopics() {
-            return referredTopics;
-        }
-
-        public void setReferredTopics(List<String> referredTopics) {
-            this.referredTopics = referredTopics;
-        }
-
-        public Message getMessageText() {
-            return messageText;
-        }
-
-        public void setMessageText(Message messageText) {
-            this.messageText = messageText;
-        }
-
-    }
-
-    public static class TwitterUser {
-
-        private String screenName;
-        private String lang = "en";
-        private int friendsCount;
-        private int statusesCount;
-        private String name;
-        private int followersCount;
-
-        public TwitterUser() {
-
-        }
-
-        public TwitterUser(String screenName, int friendsCount, int statusesCount, String name, int followersCount) {
-            this.screenName = screenName;
-            this.friendsCount = friendsCount;
-            this.statusesCount = statusesCount;
-            this.name = name;
-            this.followersCount = followersCount;
-        }
-
-        public void reset(String screenName, int friendsCount, int statusesCount, String name, int followersCount) {
-            this.screenName = screenName;
-            this.friendsCount = friendsCount;
-            this.statusesCount = statusesCount;
-            this.name = name;
-            this.followersCount = followersCount;
-        }
-
-        public String getScreenName() {
-            return screenName;
-        }
-
-        public int getFriendsCount() {
-            return friendsCount;
-        }
-
-        public int getStatusesCount() {
-            return statusesCount;
-        }
-
-        public String getName() {
-            return name;
-        }
-
-        public int getFollowersCount() {
-            return followersCount;
-        }
-
-        public String toString() {
-            StringBuilder builder = new StringBuilder();
-            builder.append("{");
-            builder.append("\"screen-name\":" + "\"" + screenName + "\"");
-            builder.append(",");
-            builder.append("\"lang\":" + "\"" + lang + "\"");
-            builder.append(",");
-            builder.append("\"friends_count\":" + friendsCount);
-            builder.append(",");
-            builder.append("\"statuses_count\":" + statusesCount);
-            builder.append(",");
-            builder.append("\"name\":" + "\"" + name + "\"");
-            builder.append(",");
-            builder.append("\"followers_count\":" + followersCount);
-            builder.append("}");
-            return builder.toString();
-        }
-
-    }
-
-    public static class DistributionHandler {
-
-        private final ZipfGenerator zipfGen;
-        private final int totalUsers;
-        private final int totalMessages;
-        private Random random = new Random();
-
-        public DistributionHandler(int totalMessages, double skew, int totalNumUsers) {
-            zipfGen = new ZipfGenerator(totalMessages, skew);
-            totalUsers = totalNumUsers;
-            this.totalMessages = totalMessages;
-        }
-
-        public int getFromDistribution(int rank) {
-            double prob = zipfGen.getProbability(rank);
-            int numMessages = (int) (prob * totalMessages);
-            return numMessages;
-        }
-
-        public static void main(String args[]) {
-            int totalMessages = 1000 * 4070;
-            double skew = 0.5;
-            int totalUsers = 4070;
-            DistributionHandler d = new DistributionHandler(totalMessages, skew, totalUsers);
-            int sum = 0;
-            for (int i = totalUsers; i >= 1; i--) {
-                float contrib = d.getFromDistribution(i);
-                sum += contrib;
-                System.out.println(i + ":" + contrib);
-            }
-
-            System.out.println("SUM" + ":" + sum);
-
-        }
-    }
-
-    public static class ZipfGenerator {
-
-        private Random rnd = new Random(System.currentTimeMillis());
-        private int size;
-        private double skew;
-        private double bottom = 0;
-
-        public ZipfGenerator(int size, double skew) {
-            this.size = size;
-            this.skew = skew;
-            for (int i = 1; i < size; i++) {
-                this.bottom += (1 / Math.pow(i, this.skew));
-            }
-        }
-
-        // the next() method returns an rank id. The frequency of returned rank
-        // ids are follows Zipf distribution.
-        public int next() {
-            int rank;
-            double friquency = 0;
-            double dice;
-            rank = rnd.nextInt(size);
-            friquency = (1.0d / Math.pow(rank, this.skew)) / this.bottom;
-            dice = rnd.nextDouble();
-            while (!(dice < friquency)) {
-                rank = rnd.nextInt(size);
-                friquency = (1.0d / Math.pow(rank, this.skew)) / this.bottom;
-                dice = rnd.nextDouble();
-            }
-            return rank;
-        }
-
-        // This method returns a probability that the given rank occurs.
-        public double getProbability(int rank) {
-            return (1.0d / Math.pow(rank, this.skew)) / this.bottom;
-        }
-
-        public static void main(String[] args) throws IOException {
-            int total = (int) (3.7 * 1000 * 1000);
-            int skew = 2;
-            int numUsers = 1000 * 1000;
-            /*
-             * if (args.length != 2) { System.out.println("usage:" +
-             * "./zipf size skew"); System.exit(-1); }
-             */
-            BufferedWriter buf = new BufferedWriter(new FileWriter(new File("/tmp/zip_output")));
-            ZipfGenerator zipf = new ZipfGenerator(total, skew);
-            double sum = 0;
-            for (int i = 1; i <= numUsers; i++) {
-                double prob = zipf.getProbability(i);
-                double contribution = (double) (prob * total);
-                String contrib = i + ":" + contribution;
-                buf.write(contrib);
-                buf.write("\n");
-                System.out.println(contrib);
-                sum += contribution;
-            }
-            System.out.println("sum is :" + sum);
-        }
-    }
-
-    public static class PartitionElement implements ILibraryElement {
-        private final String name;
-        private final String host;
-        private final int fbUserKeyMin;
-        private final int fbUserKeyMax;
-        private final int twUserKeyMin;
-        private final int twUserKeyMax;
-        private final int fbMessageIdMin;
-        private final int fbMessageIdMax;
-        private final int twMessageIdMin;
-        private final int twMessageIdMax;
-
-        public PartitionElement(String partitionName, String host, int fbUserKeyMin, int fbUserKeyMax,
-                int twUserKeyMin, int twUserKeyMax, int fbMessageIdMin, int fbMessageIdMax, int twMessageIdMin,
-                int twMessageIdMax) {
-            this.name = partitionName;
-            this.host = host;
-            this.fbUserKeyMin = fbUserKeyMin;
-            this.fbUserKeyMax = fbUserKeyMax;
-            this.twUserKeyMin = twUserKeyMax;
-            this.twUserKeyMax = twUserKeyMax;
-            this.twMessageIdMin = twMessageIdMin;
-            this.twMessageIdMax = twMessageIdMax;
-            this.fbMessageIdMin = fbMessageIdMin;
-            this.fbMessageIdMax = fbMessageIdMax;
-        }
-
-        public String toString() {
-            StringBuilder builder = new StringBuilder();
-            builder.append(name);
-            builder.append(" ");
-            builder.append(host);
-            builder.append("\n");
-            builder.append("fbUser:key:min");
-            builder.append(fbUserKeyMin);
-
-            builder.append("\n");
-            builder.append("fbUser:key:max");
-            builder.append(fbUserKeyMax);
-
-            builder.append("\n");
-            builder.append("twUser:key:min");
-            builder.append(twUserKeyMin);
-
-            builder.append("\n");
-            builder.append("twUser:key:max");
-            builder.append(twUserKeyMax);
-
-            builder.append("\n");
-            builder.append("fbMessage:key:min");
-            builder.append(fbMessageIdMin);
-
-            builder.append("\n");
-            builder.append("fbMessage:key:max");
-            builder.append(fbMessageIdMax);
-
-            builder.append("\n");
-            builder.append("twMessage:key:min");
-            builder.append(twMessageIdMin);
-
-            builder.append("\n");
-            builder.append("twMessage:key:max");
-            builder.append(twMessageIdMax);
-
-            builder.append("\n");
-            builder.append("twMessage:key:max");
-            builder.append(twUserKeyMin);
-
-            return new String(builder);
-        }
-
-        @Override
-        public String getName() {
-            return "Partition";
-        }
-
-    }
-
-    interface ILibraryElement {
-
-        public enum ElementType {
-            PARTITION
-        }
-
-        public String getName();
-
-    }
-
-    public static class Configuration {
-
-        private final float numMB;
-        private final String unit;
-
-        private final List<SourcePartition> sourcePartitions;
-        private List<TargetPartition> targetPartitions;
-
-        public Configuration(float numMB, String unit, List<SourcePartition> partitions) throws IOException {
-            this.numMB = numMB;
-            this.unit = unit;
-            this.sourcePartitions = partitions;
-
-        }
-
-        public float getNumMB() {
-            return numMB;
-        }
-
-        public String getUnit() {
-            return unit;
-        }
-
-        public List<SourcePartition> getSourcePartitions() {
-            return sourcePartitions;
-        }
-
-        public List<TargetPartition> getTargetPartitions() {
-            return targetPartitions;
-        }
-
-        public void setTargetPartitions(List<TargetPartition> targetPartitions) {
-            this.targetPartitions = targetPartitions;
-        }
-
-    }
-
-    public static class XMLUtil {
-
-        public static void writeToXML(Configuration conf, String filePath) throws IOException,
-                ParserConfigurationException, TransformerException {
-
-            DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
-            DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
-
-            // root elements
-            Document doc = docBuilder.newDocument();
-            Element rootElement = doc.createElement("Partitions");
-            doc.appendChild(rootElement);
-
-            int index = 0;
-            for (TargetPartition partition : conf.getTargetPartitions()) {
-                writePartitionElement(conf.getSourcePartitions().get(index), partition, rootElement, doc);
-            }
-
-            TransformerFactory transformerFactory = TransformerFactory.newInstance();
-            Transformer transformer = transformerFactory.newTransformer();
-
-            transformer.setOutputProperty(OutputKeys.ENCODING, "utf-8");
-            transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "no");
-            transformer.setOutputProperty(OutputKeys.INDENT, "yes");
-            transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
-
-            DOMSource source = new DOMSource(doc);
-            StreamResult result = new StreamResult(new File(filePath));
-
-            transformer.transform(source, result);
-
-        }
-
-        public static void writePartitionInfo(Configuration conf, String filePath) throws IOException {
-            BufferedWriter bw = new BufferedWriter(new FileWriter(filePath));
-            for (SourcePartition sp : conf.getSourcePartitions()) {
-                bw.write(sp.getHost() + ":" + sp.getName() + ":" + sp.getPath());
-                bw.write("\n");
-            }
-            bw.close();
-        }
-
-        public static Document getDocument(String filePath) throws Exception {
-            File inputFile = new File(filePath);
-            DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-            DocumentBuilder db = dbf.newDocumentBuilder();
-            Document doc = db.parse(inputFile);
-            doc.getDocumentElement().normalize();
-            return doc;
-        }
-
-        public static Configuration getConfiguration(String filePath) throws Exception {
-            Configuration conf = getConfiguration(getDocument(filePath));
-            PartitionMetrics metrics = new PartitionMetrics(conf.getNumMB(), conf.getUnit(), conf.getSourcePartitions()
-                    .size());
-            List<TargetPartition> targetPartitions = getTargetPartitions(metrics, conf.getSourcePartitions());
-            conf.setTargetPartitions(targetPartitions);
-            return conf;
-        }
-
-        public static Configuration getConfiguration(Document document) throws IOException {
-            Element rootEle = document.getDocumentElement();
-            NodeList nodeList = rootEle.getChildNodes();
-            float size = Float.parseFloat(getStringValue((Element) nodeList, "size"));
-            String unit = getStringValue((Element) nodeList, "unit");
-            List<SourcePartition> sourcePartitions = getSourcePartitions(document);
-            return new Configuration(size, unit, sourcePartitions);
-        }
-
-        public static List<SourcePartition> getSourcePartitions(Document document) {
-            Element rootEle = document.getDocumentElement();
-            NodeList nodeList = rootEle.getElementsByTagName("partition");
-            List<SourcePartition> sourcePartitions = new ArrayList<SourcePartition>();
-            for (int i = 0; i < nodeList.getLength(); i++) {
-                Node node = nodeList.item(i);
-                sourcePartitions.add(getSourcePartition((Element) node));
-            }
-            return sourcePartitions;
-        }
-
-        public static SourcePartition getSourcePartition(Element functionElement) {
-            String name = getStringValue(functionElement, "name");
-            String host = getStringValue(functionElement, "host");
-            String path = getStringValue(functionElement, "path");
-            SourcePartition sp = new SourcePartition(name, host, path);
-            return sp;
-        }
-
-        public static String getStringValue(Element element, String tagName) {
-            String textValue = null;
-            NodeList nl = element.getElementsByTagName(tagName);
-            if (nl != null && nl.getLength() > 0) {
-                Element el = (Element) nl.item(0);
-                textValue = el.getFirstChild().getNodeValue();
-            }
-            return textValue;
-        }
-
-        public static PartitionConfiguration getPartitionConfiguration(String filePath, String partitionName)
-                throws Exception {
-            PartitionConfiguration pconf = getPartitionConfigurations(getDocument(filePath), partitionName);
-            return pconf;
-        }
-
-        public static PartitionConfiguration getPartitionConfigurations(Document document, String partitionName)
-                throws IOException {
-
-            Element rootEle = document.getDocumentElement();
-            NodeList nodeList = rootEle.getElementsByTagName("Partition");
-
-            for (int i = 0; i < nodeList.getLength(); i++) {
-                Node node = nodeList.item(i);
-                Element nodeElement = (Element) node;
-                String name = getStringValue(nodeElement, "name");
-                if (!name.equalsIgnoreCase(partitionName)) {
-                    continue;
-                }
-                String host = getStringValue(nodeElement, "host");
-                String path = getStringValue(nodeElement, "path");
-
-                String fbUserKeyMin = getStringValue(nodeElement, "fbUserKeyMin");
-                String fbUserKeyMax = getStringValue(nodeElement, "fbUserKeyMax");
-                String twUserKeyMin = getStringValue(nodeElement, "twUserKeyMin");
-                String twUserKeyMax = getStringValue(nodeElement, "twUserKeyMax");
-                String fbMessageKeyMin = getStringValue(nodeElement, "fbMessageKeyMin");
-
-                String fbMessageKeyMax = getStringValue(nodeElement, "fbMessageKeyMax");
-                String twMessageKeyMin = getStringValue(nodeElement, "twMessageKeyMin");
-                String twMessageKeyMax = getStringValue(nodeElement, "twMessageKeyMax");
-                String numCommonUsers = getStringValue(nodeElement, "numCommonUsers");
-
-                SourcePartition sp = new SourcePartition(name, host, path);
-
-                TargetPartition tp = new TargetPartition(partitionName, host, path, Integer.parseInt(fbUserKeyMin),
-                        Integer.parseInt(fbUserKeyMax), Integer.parseInt(twUserKeyMin), Integer.parseInt(twUserKeyMax),
-                        Integer.parseInt(fbMessageKeyMin), Integer.parseInt(fbMessageKeyMax),
-                        Integer.parseInt(twMessageKeyMin), Integer.parseInt(twMessageKeyMax),
-                        Integer.parseInt(numCommonUsers));
-                PartitionConfiguration pc = new PartitionConfiguration(sp, tp);
-                return pc;
-            }
-            return null;
-        }
-
-        public static List<TargetPartition> getTargetPartitions(PartitionMetrics metrics,
-                List<SourcePartition> sourcePartitions) {
-            List<TargetPartition> partitions = new ArrayList<TargetPartition>();
-            int fbUserKeyMin = 1;
-            int twUserKeyMin = 1;
-            int fbMessageIdMin = 1;
-            int twMessageIdMin = 1;
-
-            for (SourcePartition sp : sourcePartitions) {
-                int fbUserKeyMax = fbUserKeyMin + metrics.getFbOnlyUsers() + metrics.getCommonUsers() - 1;
-                int twUserKeyMax = twUserKeyMin + metrics.getTwitterOnlyUsers() + metrics.getCommonUsers() - 1;
-
-                int fbMessageIdMax = fbMessageIdMin + metrics.getFbMessages() - 1;
-                int twMessageIdMax = twMessageIdMin + metrics.getTwMessages() - 1;
-                TargetPartition pe = new TargetPartition(sp.getName(), sp.getHost(), sp.getPath(), fbUserKeyMin,
-                        fbUserKeyMax, twUserKeyMin, twUserKeyMax, fbMessageIdMin, fbMessageIdMax, twMessageIdMin,
-                        twMessageIdMax, metrics.getCommonUsers());
-                partitions.add(pe);
-
-                fbUserKeyMin = fbUserKeyMax + 1;
-                twUserKeyMin = twUserKeyMax + 1;
-
-                fbMessageIdMin = fbMessageIdMax + 1;
-                twMessageIdMin = twMessageIdMax + 1;
-            }
-
-            return partitions;
-        }
-
-        public static void writePartitionElement(SourcePartition sourcePartition, TargetPartition partition,
-                Element rootElement, Document doc) {
-            // staff elements
-            Element pe = doc.createElement("Partition");
-            rootElement.appendChild(pe);
-
-            // name element
-            Element name = doc.createElement("name");
-            name.appendChild(doc.createTextNode("" + partition.getName()));
-            pe.appendChild(name);
-
-            // host element
-            Element host = doc.createElement("host");
-            host.appendChild(doc.createTextNode("" + partition.getHost()));
-            pe.appendChild(host);
-
-            // path element
-            Element path = doc.createElement("path");
-            path.appendChild(doc.createTextNode("" + partition.getPath()));
-            pe.appendChild(path);
-
-            // fbUserKeyMin element
-            Element fbUserKeyMin = doc.createElement("fbUserKeyMin");
-            fbUserKeyMin.appendChild(doc.createTextNode("" + partition.getFbUserKeyMin()));
-            pe.appendChild(fbUserKeyMin);
-
-            // fbUserKeyMax element
-            Element fbUserKeyMax = doc.createElement("fbUserKeyMax");
-            fbUserKeyMax.appendChild(doc.createTextNode("" + partition.getFbUserKeyMax()));
-            pe.appendChild(fbUserKeyMax);
-
-            // twUserKeyMin element
-            Element twUserKeyMin = doc.createElement("twUserKeyMin");
-            twUserKeyMin.appendChild(doc.createTextNode("" + partition.getTwUserKeyMin()));
-            pe.appendChild(twUserKeyMin);
-
-            // twUserKeyMax element
-            Element twUserKeyMax = doc.createElement("twUserKeyMax");
-            twUserKeyMax.appendChild(doc.createTextNode("" + partition.getTwUserKeyMax()));
-            pe.appendChild(twUserKeyMax);
-
-            // fbMessgeKeyMin element
-            Element fbMessageKeyMin = doc.createElement("fbMessageKeyMin");
-            fbMessageKeyMin.appendChild(doc.createTextNode("" + partition.getFbMessageIdMin()));
-            pe.appendChild(fbMessageKeyMin);
-
-            // fbMessgeKeyMin element
-            Element fbMessageKeyMax = doc.createElement("fbMessageKeyMax");
-            fbMessageKeyMax.appendChild(doc.createTextNode("" + partition.getFbMessageIdMax()));
-            pe.appendChild(fbMessageKeyMax);
-
-            // twMessgeKeyMin element
-            Element twMessageKeyMin = doc.createElement("twMessageKeyMin");
-            twMessageKeyMin.appendChild(doc.createTextNode("" + partition.getTwMessageIdMin()));
-            pe.appendChild(twMessageKeyMin);
-
-            // twMessgeKeyMin element
-            Element twMessageKeyMax = doc.createElement("twMessageKeyMax");
-            twMessageKeyMax.appendChild(doc.createTextNode("" + partition.getTwMessageIdMax()));
-            pe.appendChild(twMessageKeyMax);
-
-            // twMessgeKeyMin element
-            Element numCommonUsers = doc.createElement("numCommonUsers");
-            numCommonUsers.appendChild(doc.createTextNode("" + partition.getCommonUsers()));
-            pe.appendChild(numCommonUsers);
-
-        }
-
-        public static void main(String args[]) throws Exception {
-            String confFile = "/Users/rgrove1/work/research/asterix/icde/data-gen/conf/conf.xml";
-            String outputPath = "/Users/rgrove1/work/research/asterix/icde/data-gen/output/conf-output.xml";
-            Configuration conf = getConfiguration(confFile);
-            writeToXML(conf, outputPath);
-        }
-
-    }
-
-    public static class Date {
-
-        private int day;
-        private int month;
-        private int year;
-
-        public Date(int month, int day, int year) {
-            this.month = month;
-            this.day = day;
-            this.year = year;
-        }
-
-        public void reset(int month, int day, int year) {
-            this.month = month;
-            this.day = day;
-            this.year = year;
-        }
-
-        public int getDay() {
-            return day;
-        }
-
-        public int getMonth() {
-            return month;
-        }
-
-        public int getYear() {
-            return year;
-        }
-
-        public Date() {
-        }
-
-        public String toString() {
-            StringBuilder builder = new StringBuilder();
-            builder.append("date");
-            builder.append("(\"");
-            builder.append(year);
-            builder.append("-");
-            builder.append(month < 10 ? "0" + month : "" + month);
-            builder.append("-");
-            builder.append(day < 10 ? "0" + day : "" + day);
-            builder.append("\")");
-            return builder.toString();
-        }
-
-        public void setDay(int day) {
-            this.day = day;
-        }
-
-        public void setMonth(int month) {
-            this.month = month;
-        }
-
-        public void setYear(int year) {
-            this.year = year;
-        }
-    }
-
-    public static class PartitionMetrics {
-
-        private final int fbMessages;
-        private final int twMessages;
-
-        private final int fbOnlyUsers;
-        private final int twitterOnlyUsers;
-        private final int commonUsers;
-
-        public PartitionMetrics(float number, String unit, int numPartitions) throws IOException {
-
-            int factor = 0;
-            if (unit.equalsIgnoreCase("MB")) {
-                factor = 1024 * 1024;
-            } else if (unit.equalsIgnoreCase("GB")) {
-                factor = 1024 * 1024 * 1024;
-            } else if (unit.equalsIgnoreCase("TB")) {
-                factor = 1024 * 1024 * 1024 * 1024;
-            } else
-                throw new IOException("Invalid unit:" + unit);
-
-            fbMessages = (int) (((number * factor * 0.80) / 258) / numPartitions);
-            twMessages = (int) (fbMessages * 1.1 / 0.35);
-
-            fbOnlyUsers = (int) ((number * factor * 0.20 * 0.0043)) / numPartitions;
-            twitterOnlyUsers = (int) (0.25 * fbOnlyUsers);
-            commonUsers = (int) (0.1 * fbOnlyUsers);
-        }
-
-        public int getFbMessages() {
-            return fbMessages;
-        }
-
-        public int getTwMessages() {
-            return twMessages;
-        }
-
-        public int getFbOnlyUsers() {
-            return fbOnlyUsers;
-        }
-
-        public int getTwitterOnlyUsers() {
-            return twitterOnlyUsers;
-        }
-
-        public int getCommonUsers() {
-            return commonUsers;
-        }
-
-    }
-
-    public static String[] lastNames = { "Hoopengarner", "Harrow", "Gardner", "Blyant", "Best", "Buttermore", "Gronko",
-            "Mayers", "Countryman", "Neely", "Ruhl", "Taggart", "Bash", "Cason", "Hil", "Zalack", "Mingle", "Carr",
-            "Rohtin", "Wardle", "Pullman", "Wire", "Kellogg", "Hiles", "Keppel", "Bratton", "Sutton", "Wickes",
-            "Muller", "Friedline", "Llora", "Elizabeth", "Anderson", "Gaskins", "Rifler", "Vinsant", "Stanfield",
-            "Black", "Guest", "Hujsak", "Carter", "Weidemann", "Hays", "Patton", "Hayhurst", "Paynter", "Cressman",
-            "Fiddler", "Evans", "Sherlock", "Woodworth", "Jackson", "Bloise", "Schneider", "Ring", "Kepplinger",
-            "James", "Moon", "Bennett", "Bashline", "Ryals", "Zeal", "Christman", "Milliron", "Nash", "Ewing", "Camp",
-            "Mason", "Richardson", "Bowchiew", "Hahn", "Wilson", "Wood", "Toyley", "Williamson", "Lafortune", "Errett",
-            "Saltser", "Hirleman", "Brindle", "Newbiggin", "Ulery", "Lambert", "Shick", "Kuster", "Moore", "Finck",
-            "Powell", "Jolce", "Townsend", "Sauter", "Cowher", "Wolfe", "Cavalet", "Porter", "Laborde", "Ballou",
-            "Murray", "Stoddard", "Pycroft", "Milne", "King", "Todd", "Staymates", "Hall", "Romanoff", "Keilbach",
-            "Sandford", "Hamilton", "Fye", "Kline", "Weeks", "Mcelroy", "Mccullough", "Bryant", "Hill", "Moore",
-            "Ledgerwood", "Prevatt", "Eckert", "Read", "Hastings", "Doverspike", "Allshouse", "Bryan", "Mccallum",
-            "Lombardi", "Mckendrick", "Cattley", "Barkley", "Steiner", "Finlay", "Priebe", "Armitage", "Hall", "Elder",
-            "Erskine", "Hatcher", "Walker", "Pearsall", "Dunkle", "Haile", "Adams", "Miller", "Newbern", "Basinger",
-            "Fuhrer", "Brinigh", "Mench", "Blackburn", "Bastion", "Mccune", "Bridger", "Hynes", "Quinn", "Courtney",
-            "Geddinge", "Field", "Seelig", "Cable", "Earhart", "Harshman", "Roby", "Beals", "Berry", "Reed", "Hector",
-            "Pittman", "Haverrman", "Kalp", "Briner", "Joghs", "Cowart", "Close", "Wynne", "Harden", "Weldy",
-            "Stephenson", "Hildyard", "Moberly", "Wells", "Mackendoerfer", "Fisher", "Oppie", "Oneal", "Churchill",
-            "Keister", "Alice", "Tavoularis", "Fisher", "Hair", "Burns", "Veith", "Wile", "Fuller", "Fields", "Clark",
-            "Randolph", "Stone", "Mcclymonds", "Holtzer", "Donkin", "Wilkinson", "Rosensteel", "Albright", "Stahl",
-            "Fox", "Kadel", "Houser", "Hanseu", "Henderson", "Davis", "Bicknell", "Swain", "Mercer", "Holdeman",
-            "Enderly", "Caesar", "Margaret", "Munshower", "Elless", "Lucy", "Feufer", "Schofield", "Graham",
-            "Blatenberger", "Benford", "Akers", "Campbell", "Ann", "Sadley", "Ling", "Gongaware", "Schmidt", "Endsley",
-            "Groah", "Flanders", "Reichard", "Lowstetter", "Sandblom", "Griffis", "Basmanoff", "Coveney", "Hawker",
-            "Archibald", "Hutton", "Barnes", "Diegel", "Raybould", "Focell", "Breitenstein", "Murray", "Chauvin",
-            "Busk", "Pheleps", "Teagarden", "Northey", "Baumgartner", "Fleming", "Harris", "Parkinson", "Carpenter",
-            "Whirlow", "Bonner", "Wortman", "Rogers", "Scott", "Lowe", "Mckee", "Huston", "Bullard", "Throckmorton",
-            "Rummel", "Mathews", "Dull", "Saline", "Tue", "Woolery", "Lalty", "Schrader", "Ramsey", "Eisenmann",
-            "Philbrick", "Sybilla", "Wallace", "Fonblanque", "Paul", "Orbell", "Higgens", "Casteel", "Franks",
-            "Demuth", "Eisenman", "Hay", "Robinson", "Fischer", "Hincken", "Wylie", "Leichter", "Bousum",
-            "Littlefield", "Mcdonald", "Greif", "Rhodes", "Wall", "Steele", "Baldwin", "Smith", "Stewart", "Schere",
-            "Mary", "Aultman", "Emrick", "Guess", "Mitchell", "Painter", "Aft", "Hasely", "Weldi", "Loewentsein",
-            "Poorbaugh", "Kepple", "Noton", "Judge", "Jackson", "Style", "Adcock", "Diller", "Marriman", "Johnston",
-            "Children", "Monahan", "Ehret", "Shaw", "Congdon", "Pinney", "Millard", "Crissman", "Tanner", "Rose",
-            "Knisely", "Cypret", "Sommer", "Poehl", "Hardie", "Bender", "Overholt", "Gottwine", "Beach", "Leslie",
-            "Trevithick", "Langston", "Magor", "Shotts", "Howe", "Hunter", "Cross", "Kistler", "Dealtry", "Christner",
-            "Pennington", "Thorley", "Eckhardstein", "Van", "Stroh", "Stough", "Stall", "Beedell", "Shea", "Garland",
-            "Mays", "Pritchard", "Frankenberger", "Rowley", "Lane", "Baum", "Alliman", "Park", "Jardine", "Butler",
-            "Cherry", "Kooser", "Baxter", "Billimek", "Downing", "Hurst", "Wood", "Baird", "Watkins", "Edwards",
-            "Kemerer", "Harding", "Owens", "Eiford", "Keener", "Garneis", "Fiscina", "Mang", "Draudy", "Mills",
-            "Gibson", "Reese", "Todd", "Ramos", "Levett", "Wilks", "Ward", "Mosser", "Dunlap", "Kifer", "Christopher",
-            "Ashbaugh", "Wynter", "Rawls", "Cribbs", "Haynes", "Thigpen", "Schreckengost", "Bishop", "Linton",
-            "Chapman", "James", "Jerome", "Hook", "Omara", "Houston", "Maclagan", "Sandys", "Pickering", "Blois",
-            "Dickson", "Kemble", "Duncan", "Woodward", "Southern", "Henley", "Treeby", "Cram", "Elsas", "Driggers",
-            "Warrick", "Overstreet", "Hindman", "Buck", "Sulyard", "Wentzel", "Swink", "Butt", "Schaeffer",
-            "Hoffhants", "Bould", "Willcox", "Lotherington", "Bagley", "Graff", "White", "Wheeler", "Sloan",
-            "Rodacker", "Hanford", "Jowers", "Kunkle", "Cass", "Powers", "Gilman", "Mcmichaels", "Hobbs", "Herndon",
-            "Prescott", "Smail", "Mcdonald", "Biery", "Orner", "Richards", "Mueller", "Isaman", "Bruxner", "Goodman",
-            "Barth", "Turzanski", "Vorrasi", "Stainforth", "Nehling", "Rahl", "Erschoff", "Greene", "Mckinnon",
-            "Reade", "Smith", "Pery", "Roose", "Greenwood", "Weisgarber", "Curry", "Holts", "Zadovsky", "Parrish",
-            "Putnam", "Munson", "Mcindoe", "Nickolson", "Brooks", "Bollinger", "Stroble", "Siegrist", "Fulton",
-            "Tomey", "Zoucks", "Roberts", "Otis", "Clarke", "Easter", "Johnson", "Fylbrigg", "Taylor", "Swartzbaugh",
-            "Weinstein", "Gadow", "Sayre", "Marcotte", "Wise", "Atweeke", "Mcfall", "Napier", "Eisenhart", "Canham",
-            "Sealis", "Baughman", "Gertraht", "Losey", "Laurence", "Eva", "Pershing", "Kern", "Pirl", "Rega",
-            "Sanborn", "Kanaga", "Sanders", "Anderson", "Dickinson", "Osteen", "Gettemy", "Crom", "Snyder", "Reed",
-            "Laurenzi", "Riggle", "Tillson", "Fowler", "Raub", "Jenner", "Koepple", "Soames", "Goldvogel", "Dimsdale",
-            "Zimmer", "Giesen", "Baker", "Beail", "Mortland", "Bard", "Sanner", "Knopsnider", "Jenkins", "Bailey",
-            "Werner", "Barrett", "Faust", "Agg", "Tomlinson", "Williams", "Little", "Greenawalt", "Wells", "Wilkins",
-            "Gisiko", "Bauerle", "Harrold", "Prechtl", "Polson", "Faast", "Winton", "Garneys", "Peters", "Potter",
-            "Porter", "Tennant", "Eve", "Dugger", "Jones", "Burch", "Cowper", "Whittier" };
-
-    public static String[] firstNames = { "Albert", "Jacquelin", "Dona", "Alia", "Mayme", "Genoveva", "Emma", "Lena",
-            "Melody", "Vilma", "Katelyn", "Jeremy", "Coral", "Leann", "Lita", "Gilda", "Kayla", "Alvina", "Maranda",
-            "Verlie", "Khadijah", "Karey", "Patrice", "Kallie", "Corey", "Mollie", "Daisy", "Melanie", "Sarita",
-            "Nichole", "Pricilla", "Terresa", "Berneice", "Arianne", "Brianne", "Lavinia", "Ulrike", "Lesha", "Adell",
-            "Ardelle", "Marisha", "Laquita", "Karyl", "Maryjane", "Kendall", "Isobel", "Raeann", "Heike", "Barbera",
-            "Norman", "Yasmine", "Nevada", "Mariam", "Edith", "Eugena", "Lovie", "Maren", "Bennie", "Lennie", "Tamera",
-            "Crystal", "Randi", "Anamaria", "Chantal", "Jesenia", "Avis", "Shela", "Randy", "Laurena", "Sharron",
-            "Christiane", "Lorie", "Mario", "Elizabeth", "Reina", "Adria", "Lakisha", "Brittni", "Azzie", "Dori",
-            "Shaneka", "Asuncion", "Katheryn", "Laurice", "Sharita", "Krystal", "Reva", "Inger", "Alpha", "Makeda",
-            "Anabel", "Loni", "Tiara", "Meda", "Latashia", "Leola", "Chin", "Daisey", "Ivory", "Amalia", "Logan",
-            "Tyler", "Kyong", "Carolann", "Maryetta", "Eufemia", "Anya", "Doreatha", "Lorna", "Rutha", "Ehtel",
-            "Debbie", "Chassidy", "Sang", "Christa", "Lottie", "Chun", "Karine", "Peggie", "Amina", "Melany", "Alayna",
-            "Scott", "Romana", "Naomi", "Christiana", "Salena", "Taunya", "Mitsue", "Regina", "Chelsie", "Charity",
-            "Dacia", "Aletha", "Latosha", "Lia", "Tamica", "Chery", "Bianca", "Shu", "Georgianne", "Myriam", "Austin",
-            "Wan", "Mallory", "Jana", "Georgie", "Jenell", "Kori", "Vicki", "Delfina", "June", "Mellisa", "Catherina",
-            "Claudie", "Tynisha", "Dayle", "Enriqueta", "Belen", "Pia", "Sarai", "Rosy", "Renay", "Kacie", "Frieda",
-            "Cayla", "Elissa", "Claribel", "Sabina", "Mackenzie", "Raina", "Cira", "Mitzie", "Aubrey", "Serafina",
-            "Maria", "Katharine", "Esperanza", "Sung", "Daria", "Billye", "Stefanie", "Kasha", "Holly", "Suzanne",
-            "Inga", "Flora", "Andria", "Genevie", "Eladia", "Janet", "Erline", "Renna", "Georgeanna", "Delorse",
-            "Elnora", "Rudy", "Rima", "Leanora", "Letisha", "Love", "Alverta", "Pinkie", "Domonique", "Jeannie",
-            "Jose", "Jacqueline", "Tara", "Lily", "Erna", "Tennille", "Galina", "Tamala", "Kirby", "Nichelle",
-            "Myesha", "Farah", "Santa", "Ludie", "Kenia", "Yee", "Micheline", "Maryann", "Elaina", "Ethelyn",
-            "Emmaline", "Shanell", "Marina", "Nila", "Alane", "Shakira", "Dorris", "Belinda", "Elois", "Barbie",
-            "Carita", "Gisela", "Lura", "Fransisca", "Helga", "Peg", "Leonarda", "Earlie", "Deetta", "Jacquetta",
-            "Blossom", "Kayleigh", "Deloras", "Keshia", "Christinia", "Dulce", "Bernie", "Sheba", "Lashanda", "Tula",
-            "Claretta", "Kary", "Jeanette", "Lupita", "Lenora", "Hisako", "Sherise", "Glynda", "Adela", "Chia",
-            "Sudie", "Mindy", "Caroyln", "Lindsey", "Xiomara", "Mercedes", "Onie", "Loan", "Alexis", "Tommie",
-            "Donette", "Monica", "Soo", "Camellia", "Lavera", "Valery", "Ariana", "Sophia", "Loris", "Ginette",
-            "Marielle", "Tari", "Julissa", "Alesia", "Suzanna", "Emelda", "Erin", "Ladawn", "Sherilyn", "Candice",
-            "Nereida", "Fairy", "Carl", "Joel", "Marilee", "Gracia", "Cordie", "So", "Shanita", "Drew", "Cassie",
-            "Sherie", "Marget", "Norma", "Delois", "Debera", "Chanelle", "Catarina", "Aracely", "Carlene", "Tricia",
-            "Aleen", "Katharina", "Marguerita", "Guadalupe", "Margorie", "Mandie", "Kathe", "Chong", "Sage", "Faith",
-            "Maryrose", "Stephany", "Ivy", "Pauline", "Susie", "Cristen", "Jenifer", "Annette", "Debi", "Karmen",
-            "Luci", "Shayla", "Hope", "Ocie", "Sharie", "Tami", "Breana", "Kerry", "Rubye", "Lashay", "Sondra",
-            "Katrice", "Brunilda", "Cortney", "Yan", "Zenobia", "Penni", "Addie", "Lavona", "Noel", "Anika",
-            "Herlinda", "Valencia", "Bunny", "Tory", "Victoria", "Carrie", "Mikaela", "Wilhelmina", "Chung",
-            "Hortencia", "Gerda", "Wen", "Ilana", "Sibyl", "Candida", "Victorina", "Chantell", "Casie", "Emeline",
-            "Dominica", "Cecila", "Delora", "Miesha", "Nova", "Sally", "Ronald", "Charlette", "Francisca", "Mina",
-            "Jenna", "Loraine", "Felisa", "Lulu", "Page", "Lyda", "Babara", "Flor", "Walter", "Chan", "Sherika",
-            "Kala", "Luna", "Vada", "Syreeta", "Slyvia", "Karin", "Renata", "Robbi", "Glenda", "Delsie", "Lizzie",
-            "Genia", "Caitlin", "Bebe", "Cory", "Sam", "Leslee", "Elva", "Caren", "Kasie", "Leticia", "Shannan",
-            "Vickey", "Sandie", "Kyle", "Chang", "Terrilyn", "Sandra", "Elida", "Marketta", "Elsy", "Tu", "Carman",
-            "Ashlie", "Vernia", "Albertine", "Vivian", "Elba", "Bong", "Margy", "Janetta", "Xiao", "Teofila", "Danyel",
-            "Nickole", "Aleisha", "Tera", "Cleotilde", "Dara", "Paulita", "Isela", "Maricela", "Rozella", "Marivel",
-            "Aurora", "Melissa", "Carylon", "Delinda", "Marvella", "Candelaria", "Deidre", "Tawanna", "Myrtie",
-            "Milagro", "Emilie", "Coretta", "Ivette", "Suzann", "Ammie", "Lucina", "Lory", "Tena", "Eleanor",
-            "Cherlyn", "Tiana", "Brianna", "Myra", "Flo", "Carisa", "Kandi", "Erlinda", "Jacqulyn", "Fermina", "Riva",
-            "Palmira", "Lindsay", "Annmarie", "Tamiko", "Carline", "Amelia", "Quiana", "Lashawna", "Veola", "Belva",
-            "Marsha", "Verlene", "Alex", "Leisha", "Camila", "Mirtha", "Melva", "Lina", "Arla", "Cythia", "Towanda",
-            "Aracelis", "Tasia", "Aurore", "Trinity", "Bernadine", "Farrah", "Deneen", "Ines", "Betty", "Lorretta",
-            "Dorethea", "Hertha", "Rochelle", "Juli", "Shenika", "Yung", "Lavon", "Deeanna", "Nakia", "Lynnette",
-            "Dinorah", "Nery", "Elene", "Carolee", "Mira", "Franchesca", "Lavonda", "Leida", "Paulette", "Dorine",
-            "Allegra", "Keva", "Jeffrey", "Bernardina", "Maryln", "Yoko", "Faviola", "Jayne", "Lucilla", "Charita",
-            "Ewa", "Ella", "Maggie", "Ivey", "Bettie", "Jerri", "Marni", "Bibi", "Sabrina", "Sarah", "Marleen",
-            "Katherin", "Remona", "Jamika", "Antonina", "Oliva", "Lajuana", "Fonda", "Sigrid", "Yael", "Billi",
-            "Verona", "Arminda", "Mirna", "Tesha", "Katheleen", "Bonita", "Kamilah", "Patrica", "Julio", "Shaina",
-            "Mellie", "Denyse", "Deandrea", "Alena", "Meg", "Kizzie", "Krissy", "Karly", "Alleen", "Yahaira", "Lucie",
-            "Karena", "Elaine", "Eloise", "Buena", "Marianela", "Renee", "Nan", "Carolynn", "Windy", "Avril", "Jane",
-            "Vida", "Thea", "Marvel", "Rosaline", "Tifany", "Robena", "Azucena", "Carlota", "Mindi", "Andera", "Jenny",
-            "Courtney", "Lyndsey", "Willette", "Kristie", "Shaniqua", "Tabatha", "Ngoc", "Una", "Marlena", "Louetta",
-            "Vernie", "Brandy", "Jacquelyne", "Jenelle", "Elna", "Erminia", "Ida", "Audie", "Louis", "Marisol",
-            "Shawana", "Harriette", "Karol", "Kitty", "Esmeralda", "Vivienne", "Eloisa", "Iris", "Jeanice", "Cammie",
-            "Jacinda", "Shena", "Floy", "Theda", "Lourdes", "Jayna", "Marg", "Kati", "Tanna", "Rosalyn", "Maxima",
-            "Soon", "Angelika", "Shonna", "Merle", "Kassandra", "Deedee", "Heidi", "Marti", "Renae", "Arleen",
-            "Alfredia", "Jewell", "Carley", "Pennie", "Corina", "Tonisha", "Natividad", "Lilliana", "Darcie", "Shawna",
-            "Angel", "Piedad", "Josefa", "Rebbeca", "Natacha", "Nenita", "Petrina", "Carmon", "Chasidy", "Temika",
-            "Dennise", "Renetta", "Augusta", "Shirlee", "Valeri", "Casimira", "Janay", "Berniece", "Deborah", "Yaeko",
-            "Mimi", "Digna", "Irish", "Cher", "Yong", "Lucila", "Jimmie", "Junko", "Lezlie", "Waneta", "Sandee",
-            "Marquita", "Eura", "Freeda", "Annabell", "Laree", "Jaye", "Wendy", "Toshia", "Kylee", "Aleta", "Emiko",
-            "Clorinda", "Sixta", "Audrea", "Juanita", "Birdie", "Reita", "Latanya", "Nia", "Leora", "Laurine",
-            "Krysten", "Jerrie", "Chantel", "Ira", "Sena", "Andre", "Jann", "Marla", "Precious", "Katy", "Gabrielle",
-            "Yvette", "Brook", "Shirlene", "Eldora", "Laura", "Milda", "Euna", "Jettie", "Debora", "Lise", "Edythe",
-            "Leandra", "Shandi", "Araceli", "Johanne", "Nieves", "Denese", "Carmelita", "Nohemi", "Annice", "Natalie",
-            "Yolande", "Jeffie", "Vashti", "Vickie", "Obdulia", "Youlanda", "Lupe", "Tomoko", "Monserrate", "Domitila",
-            "Etsuko", "Adrienne", "Lakesha", "Melissia", "Odessa", "Meagan", "Veronika", "Jolyn", "Isabelle", "Leah",
-            "Rhiannon", "Gianna", "Audra", "Sommer", "Renate", "Perla", "Thao", "Myong", "Lavette", "Mark", "Emilia",
-            "Ariane", "Karl", "Dorie", "Jacquie", "Mia", "Malka", "Shenita", "Tashina", "Christine", "Cherri", "Roni",
-            "Fran", "Mildred", "Sara", "Clarissa", "Fredia", "Elease", "Samuel", "Earlene", "Vernita", "Mae", "Concha",
-            "Renea", "Tamekia", "Hye", "Ingeborg", "Tessa", "Kelly", "Kristin", "Tam", "Sacha", "Kanisha", "Jillian",
-            "Tiffanie", "Ashlee", "Madelyn", "Donya", "Clementine", "Mickie", "My", "Zena", "Terrie", "Samatha",
-            "Gertie", "Tarra", "Natalia", "Sharlene", "Evie", "Shalon", "Rosalee", "Numbers", "Jodi", "Hattie",
-            "Naoma", "Valene", "Whitley", "Claude", "Alline", "Jeanne", "Camie", "Maragret", "Viola", "Kris", "Marlo",
-            "Arcelia", "Shari", "Jalisa", "Corrie", "Eleonor", "Angelyn", "Merry", "Lauren", "Melita", "Gita",
-            "Elenor", "Aurelia", "Janae", "Lyndia", "Margeret", "Shawanda", "Rolande", "Shirl", "Madeleine", "Celinda",
-            "Jaleesa", "Shemika", "Joye", "Tisa", "Trudie", "Kathrine", "Clarita", "Dinah", "Georgia", "Antoinette",
-            "Janis", "Suzette", "Sherri", "Herta", "Arie", "Hedy", "Cassi", "Audrie", "Caryl", "Jazmine", "Jessica",
-            "Beverly", "Elizbeth", "Marylee", "Londa", "Fredericka", "Argelia", "Nana", "Donnette", "Damaris",
-            "Hailey", "Jamee", "Kathlene", "Glayds", "Lydia", "Apryl", "Verla", "Adam", "Concepcion", "Zelda",
-            "Shonta", "Vernice", "Detra", "Meghann", "Sherley", "Sheri", "Kiyoko", "Margarita", "Adaline", "Mariela",
-            "Velda", "Ailene", "Juliane", "Aiko", "Edyth", "Cecelia", "Shavon", "Florance", "Madeline", "Rheba",
-            "Deann", "Ignacia", "Odelia", "Heide", "Mica", "Jennette", "Maricruz", "Ouida", "Darcy", "Laure",
-            "Justina", "Amada", "Laine", "Cruz", "Sunny", "Francene", "Roxanna", "Nam", "Nancie", "Deanna", "Letty",
-            "Britni", "Kazuko", "Lacresha", "Simon", "Caleb", "Milton", "Colton", "Travis", "Miles", "Jonathan",
-            "Logan", "Rolf", "Emilio", "Roberto", "Marcus", "Tim", "Delmar", "Devon", "Kurt", "Edward", "Jeffrey",
-            "Elvis", "Alfonso", "Blair", "Wm", "Sheldon", "Leonel", "Michal", "Federico", "Jacques", "Leslie",
-            "Augustine", "Hugh", "Brant", "Hong", "Sal", "Modesto", "Curtis", "Jefferey", "Adam", "John", "Glenn",
-            "Vance", "Alejandro", "Refugio", "Lucio", "Demarcus", "Chang", "Huey", "Neville", "Preston", "Bert",
-            "Abram", "Foster", "Jamison", "Kirby", "Erich", "Manual", "Dustin", "Derrick", "Donnie", "Jospeh", "Chris",
-            "Josue", "Stevie", "Russ", "Stanley", "Nicolas", "Samuel", "Waldo", "Jake", "Max", "Ernest", "Reinaldo",
-            "Rene", "Gale", "Morris", "Nathan", "Maximo", "Courtney", "Theodore", "Octavio", "Otha", "Delmer",
-            "Graham", "Dean", "Lowell", "Myles", "Colby", "Boyd", "Adolph", "Jarrod", "Nick", "Mark", "Clinton", "Kim",
-            "Sonny", "Dalton", "Tyler", "Jody", "Orville", "Luther", "Rubin", "Hollis", "Rashad", "Barton", "Vicente",
-            "Ted", "Rick", "Carmine", "Clifton", "Gayle", "Christopher", "Jessie", "Bradley", "Clay", "Theo", "Josh",
-            "Mitchell", "Boyce", "Chung", "Eugenio", "August", "Norbert", "Sammie", "Jerry", "Adan", "Edmundo",
-            "Homer", "Hilton", "Tod", "Kirk", "Emmett", "Milan", "Quincy", "Jewell", "Herb", "Steve", "Carmen",
-            "Bobby", "Odis", "Daron", "Jeremy", "Carl", "Hunter", "Tuan", "Thurman", "Asa", "Brenton", "Shane",
-            "Donny", "Andreas", "Teddy", "Dario", "Cyril", "Hoyt", "Teodoro", "Vincenzo", "Hilario", "Daren",
-            "Agustin", "Marquis", "Ezekiel", "Brendan", "Johnson", "Alden", "Richie", "Granville", "Chad", "Joseph",
-            "Lamont", "Jordon", "Gilberto", "Chong", "Rosendo", "Eddy", "Rob", "Dewitt", "Andre", "Titus", "Russell",
-            "Rigoberto", "Dick", "Garland", "Gabriel", "Hank", "Darius", "Ignacio", "Lazaro", "Johnie", "Mauro",
-            "Edmund", "Trent", "Harris", "Osvaldo", "Marvin", "Judson", "Rodney", "Randall", "Renato", "Richard",
-            "Denny", "Jon", "Doyle", "Cristopher", "Wilson", "Christian", "Jamie", "Roland", "Ken", "Tad", "Romeo",
-            "Seth", "Quinton", "Byron", "Ruben", "Darrel", "Deandre", "Broderick", "Harold", "Ty", "Monroe", "Landon",
-            "Mohammed", "Angel", "Arlen", "Elias", "Andres", "Carlton", "Numbers", "Tony", "Thaddeus", "Issac",
-            "Elmer", "Antoine", "Ned", "Fermin", "Grover", "Benito", "Abdul", "Cortez", "Eric", "Maxwell", "Coy",
-            "Gavin", "Rich", "Andy", "Del", "Giovanni", "Major", "Efren", "Horacio", "Joaquin", "Charles", "Noah",
-            "Deon", "Pasquale", "Reed", "Fausto", "Jermaine", "Irvin", "Ray", "Tobias", "Carter", "Yong", "Jorge",
-            "Brent", "Daniel", "Zane", "Walker", "Thad", "Shaun", "Jaime", "Mckinley", "Bradford", "Nathanial",
-            "Jerald", "Aubrey", "Virgil", "Abel", "Philip", "Chester", "Chadwick", "Dominick", "Britt", "Emmitt",
-            "Ferdinand", "Julian", "Reid", "Santos", "Dwain", "Morgan", "James", "Marion", "Micheal", "Eddie", "Brett",
-            "Stacy", "Kerry", "Dale", "Nicholas", "Darrick", "Freeman", "Scott", "Newton", "Sherman", "Felton",
-            "Cedrick", "Winfred", "Brad", "Fredric", "Dewayne", "Virgilio", "Reggie", "Edgar", "Heriberto", "Shad",
-            "Timmy", "Javier", "Nestor", "Royal", "Lynn", "Irwin", "Ismael", "Jonas", "Wiley", "Austin", "Kieth",
-            "Gonzalo", "Paris", "Earnest", "Arron", "Jarred", "Todd", "Erik", "Maria", "Chauncey", "Neil", "Conrad",
-            "Maurice", "Roosevelt", "Jacob", "Sydney", "Lee", "Basil", "Louis", "Rodolfo", "Rodger", "Roman", "Corey",
-            "Ambrose", "Cristobal", "Sylvester", "Benton", "Franklin", "Marcelo", "Guillermo", "Toby", "Jeramy",
-            "Donn", "Danny", "Dwight", "Clifford", "Valentine", "Matt", "Jules", "Kareem", "Ronny", "Lonny", "Son",
-            "Leopoldo", "Dannie", "Gregg", "Dillon", "Orlando", "Weston", "Kermit", "Damian", "Abraham", "Walton",
-            "Adrian", "Rudolf", "Will", "Les", "Norberto", "Fred", "Tyrone", "Ariel", "Terry", "Emmanuel", "Anderson",
-            "Elton", "Otis", "Derek", "Frankie", "Gino", "Lavern", "Jarod", "Kenny", "Dane", "Keenan", "Bryant",
-            "Eusebio", "Dorian", "Ali", "Lucas", "Wilford", "Jeremiah", "Warner", "Woodrow", "Galen", "Bob",
-            "Johnathon", "Amado", "Michel", "Harry", "Zachery", "Taylor", "Booker", "Hershel", "Mohammad", "Darrell",
-            "Kyle", "Stuart", "Marlin", "Hyman", "Jeffery", "Sidney", "Merrill", "Roy", "Garrett", "Porter", "Kenton",
-            "Giuseppe", "Terrance", "Trey", "Felix", "Buster", "Von", "Jackie", "Linwood", "Darron", "Francisco",
-            "Bernie", "Diego", "Brendon", "Cody", "Marco", "Ahmed", "Antonio", "Vince", "Brooks", "Kendrick", "Ross",
-            "Mohamed", "Jim", "Benny", "Gerald", "Pablo", "Charlie", "Antony", "Werner", "Hipolito", "Minh", "Mel",
-            "Derick", "Armand", "Fidel", "Lewis", "Donnell", "Desmond", "Vaughn", "Guadalupe", "Keneth", "Rodrick",
-            "Spencer", "Chas", "Gus", "Harlan", "Wes", "Carmelo", "Jefferson", "Gerard", "Jarvis", "Haywood", "Hayden",
-            "Sergio", "Gene", "Edgardo", "Colin", "Horace", "Dominic", "Aldo", "Adolfo", "Juan", "Man", "Lenard",
-            "Clement", "Everett", "Hal", "Bryon", "Mason", "Emerson", "Earle", "Laurence", "Columbus", "Lamar",
-            "Douglas", "Ian", "Fredrick", "Marc", "Loren", "Wallace", "Randell", "Noble", "Ricardo", "Rory", "Lindsey",
-            "Boris", "Bill", "Carlos", "Domingo", "Grant", "Craig", "Ezra", "Matthew", "Van", "Rudy", "Danial",
-            "Brock", "Maynard", "Vincent", "Cole", "Damion", "Ellsworth", "Marcel", "Markus", "Rueben", "Tanner",
-            "Reyes", "Hung", "Kennith", "Lindsay", "Howard", "Ralph", "Jed", "Monte", "Garfield", "Avery", "Bernardo",
-            "Malcolm", "Sterling", "Ezequiel", "Kristofer", "Luciano", "Casey", "Rosario", "Ellis", "Quintin",
-            "Trevor", "Miquel", "Jordan", "Arthur", "Carson", "Tyron", "Grady", "Walter", "Jonathon", "Ricky",
-            "Bennie", "Terrence", "Dion", "Dusty", "Roderick", "Isaac", "Rodrigo", "Harrison", "Zack", "Dee", "Devin",
-            "Rey", "Ulysses", "Clint", "Greg", "Dino", "Frances", "Wade", "Franklyn", "Jude", "Bradly", "Salvador",
-            "Rocky", "Weldon", "Lloyd", "Milford", "Clarence", "Alec", "Allan", "Bobbie", "Oswaldo", "Wilfred",
-            "Raleigh", "Shelby", "Willy", "Alphonso", "Arnoldo", "Robbie", "Truman", "Nicky", "Quinn", "Damien",
-            "Lacy", "Marcos", "Parker", "Burt", "Carroll", "Denver", "Buck", "Dong", "Normand", "Billie", "Edwin",
-            "Troy", "Arden", "Rusty", "Tommy", "Kenneth", "Leo", "Claud", "Joel", "Kendall", "Dante", "Milo", "Cruz",
-            "Lucien", "Ramon", "Jarrett", "Scottie", "Deshawn", "Ronnie", "Pete", "Alonzo", "Whitney", "Stefan",
-            "Sebastian", "Edmond", "Enrique", "Branden", "Leonard", "Loyd", "Olin", "Ron", "Rhett", "Frederic",
-            "Orval", "Tyrell", "Gail", "Eli", "Antonia", "Malcom", "Sandy", "Stacey", "Nickolas", "Hosea", "Santo",
-            "Oscar", "Fletcher", "Dave", "Patrick", "Dewey", "Bo", "Vito", "Blaine", "Randy", "Robin", "Winston",
-            "Sammy", "Edwardo", "Manuel", "Valentin", "Stanford", "Filiberto", "Buddy", "Zachariah", "Johnnie",
-            "Elbert", "Paul", "Isreal", "Jerrold", "Leif", "Owen", "Sung", "Junior", "Raphael", "Josef", "Donte",
-            "Allen", "Florencio", "Raymond", "Lauren", "Collin", "Eliseo", "Bruno", "Martin", "Lyndon", "Kurtis",
-            "Salvatore", "Erwin", "Michael", "Sean", "Davis", "Alberto", "King", "Rolland", "Joe", "Tory", "Chase",
-            "Dallas", "Vernon", "Beau", "Terrell", "Reynaldo", "Monty", "Jame", "Dirk", "Florentino", "Reuben", "Saul",
-            "Emory", "Esteban", "Michale", "Claudio", "Jacinto", "Kelley", "Levi", "Andrea", "Lanny", "Wendell",
-            "Elwood", "Joan", "Felipe", "Palmer", "Elmo", "Lawrence", "Hubert", "Rudolph", "Duane", "Cordell",
-            "Everette", "Mack", "Alan", "Efrain", "Trenton", "Bryan", "Tom", "Wilmer", "Clyde", "Chance", "Lou",
-            "Brain", "Justin", "Phil", "Jerrod", "George", "Kris", "Cyrus", "Emery", "Rickey", "Lincoln", "Renaldo",
-            "Mathew", "Luke", "Dwayne", "Alexis", "Jackson", "Gil", "Marty", "Burton", "Emil", "Glen", "Willian",
-            "Clemente", "Keven", "Barney", "Odell", "Reginald", "Aurelio", "Damon", "Ward", "Gustavo", "Harley",
-            "Peter", "Anibal", "Arlie", "Nigel", "Oren", "Zachary", "Scot", "Bud", "Wilbert", "Bart", "Josiah",
-            "Marlon", "Eldon", "Darryl", "Roger", "Anthony", "Omer", "Francis", "Patricia", "Moises", "Chuck",
-            "Waylon", "Hector", "Jamaal", "Cesar", "Julius", "Rex", "Norris", "Ollie", "Isaias", "Quentin", "Graig",
-            "Lyle", "Jeffry", "Karl", "Lester", "Danilo", "Mike", "Dylan", "Carlo", "Ryan", "Leon", "Percy", "Lucius",
-            "Jamel", "Lesley", "Joey", "Cornelius", "Rico", "Arnulfo", "Chet", "Margarito", "Ernie", "Nathanael",
-            "Amos", "Cleveland", "Luigi", "Alfonzo", "Phillip", "Clair", "Elroy", "Alva", "Hans", "Shon", "Gary",
-            "Jesus", "Cary", "Silas", "Keith", "Israel", "Willard", "Randolph", "Dan", "Adalberto", "Claude",
-            "Delbert", "Garry", "Mary", "Larry", "Riley", "Robt", "Darwin", "Barrett", "Steven", "Kelly", "Herschel",
-            "Darnell", "Scotty", "Armando", "Miguel", "Lawerence", "Wesley", "Garth", "Carol", "Micah", "Alvin",
-            "Billy", "Earl", "Pat", "Brady", "Cory", "Carey", "Bernard", "Jayson", "Nathaniel", "Gaylord", "Archie",
-            "Dorsey", "Erasmo", "Angelo", "Elisha", "Long", "Augustus", "Hobert", "Drew", "Stan", "Sherwood",
-            "Lorenzo", "Forrest", "Shawn", "Leigh", "Hiram", "Leonardo", "Gerry", "Myron", "Hugo", "Alvaro", "Leland",
-            "Genaro", "Jamey", "Stewart", "Elden", "Irving", "Olen", "Antone", "Freddy", "Lupe", "Joshua", "Gregory",
-            "Andrew", "Sang", "Wilbur", "Gerardo", "Merlin", "Williams", "Johnny", "Alex", "Tommie", "Jimmy",
-            "Donovan", "Dexter", "Gaston", "Tracy", "Jeff", "Stephen", "Berry", "Anton", "Darell", "Fritz", "Willis",
-            "Noel", "Mariano", "Crawford", "Zoey", "Alex", "Brianna", "Carlie", "Lloyd", "Cal", "Astor", "Randolf",
-            "Magdalene", "Trevelyan", "Terance", "Roy", "Kermit", "Harriett", "Crystal", "Laurinda", "Kiersten",
-            "Phyllida", "Liz", "Bettie", "Rena", "Colten", "Berenice", "Sindy", "Wilma", "Amos", "Candi", "Ritchie",
-            "Dirk", "Kathlyn", "Callista", "Anona", "Flossie", "Sterling", "Calista", "Regan", "Erica", "Jeana",
-            "Keaton", "York", "Nolan", "Daniel", "Benton", "Tommie", "Serenity", "Deanna", "Chas", "Heron", "Marlyn",
-            "Xylia", "Tristin", "Lyndon", "Andriana", "Madelaine", "Maddison", "Leila", "Chantelle", "Audrey",
-            "Connor", "Daley", "Tracee", "Tilda", "Eliot", "Merle", "Linwood", "Kathryn", "Silas", "Alvina",
-            "Phinehas", "Janis", "Alvena", "Zubin", "Gwendolen", "Caitlyn", "Bertram", "Hailee", "Idelle", "Homer",
-            "Jannah", "Delbert", "Rhianna", "Cy", "Jefferson", "Wayland", "Nona", "Tempest", "Reed", "Jenifer",
-            "Ellery", "Nicolina", "Aldous", "Prince", "Lexia", "Vinnie", "Doug", "Alberic", "Kayleen", "Woody",
-            "Rosanne", "Ysabel", "Skyler", "Twyla", "Geordie", "Leta", "Clive", "Aaron", "Scottie", "Celeste", "Chuck",
-            "Erle", "Lallie", "Jaycob", "Ray", "Carrie", "Laurita", "Noreen", "Meaghan", "Ulysses", "Andy", "Drogo",
-            "Dina", "Yasmin", "Mya", "Luvenia", "Urban", "Jacob", "Laetitia", "Sherry", "Love", "Michaela", "Deonne",
-            "Summer", "Brendon", "Sheena", "Mason", "Jayson", "Linden", "Salal", "Darrell", "Diana", "Hudson",
-            "Lennon", "Isador", "Charley", "April", "Ralph", "James", "Mina", "Jolyon", "Laurine", "Monna", "Carita",
-            "Munro", "Elsdon", "Everette", "Radclyffe", "Darrin", "Herbert", "Gawain", "Sheree", "Trudy", "Emmaline",
-            "Kassandra", "Rebecca", "Basil", "Jen", "Don", "Osborne", "Lilith", "Hannah", "Fox", "Rupert", "Paulene",
-            "Darius", "Wally", "Baptist", "Sapphire", "Tia", "Sondra", "Kylee", "Ashton", "Jepson", "Joetta", "Val",
-            "Adela", "Zacharias", "Zola", "Marmaduke", "Shannah", "Posie", "Oralie", "Brittany", "Ernesta", "Raymund",
-            "Denzil", "Daren", "Roosevelt", "Nelson", "Fortune", "Mariel", "Nick", "Jaden", "Upton", "Oz", "Margaux",
-            "Precious", "Albert", "Bridger", "Jimmy", "Nicola", "Rosalynne", "Keith", "Walt", "Della", "Joanna",
-            "Xenia", "Esmeralda", "Major", "Simon", "Rexana", "Stacy", "Calanthe", "Sherley", "Kaitlyn", "Graham",
-            "Ramsey", "Abbey", "Madlyn", "Kelvin", "Bill", "Rue", "Monica", "Caileigh", "Laraine", "Booker", "Jayna",
-            "Greta", "Jervis", "Sherman", "Kendrick", "Tommy", "Iris", "Geffrey", "Kaelea", "Kerr", "Garrick", "Jep",
-            "Audley", "Nic", "Bronte", "Beulah", "Patricia", "Jewell", "Deidra", "Cory", "Everett", "Harper",
-            "Charity", "Godfrey", "Jaime", "Sinclair", "Talbot", "Dayna", "Cooper", "Rosaline", "Jennie", "Eileen",
-            "Latanya", "Corinna", "Roxie", "Caesar", "Charles", "Pollie", "Lindsey", "Sorrel", "Dwight", "Jocelyn",
-            "Weston", "Shyla", "Valorie", "Bessie", "Josh", "Lessie", "Dayton", "Kathi", "Chasity", "Wilton", "Adam",
-            "William", "Ash", "Angela", "Ivor", "Ria", "Jazmine", "Hailey", "Jo", "Silvestra", "Ernie", "Clifford",
-            "Levi", "Matilda", "Quincey", "Camilla", "Delicia", "Phemie", "Laurena", "Bambi", "Lourdes", "Royston",
-            "Chastity", "Lynwood", "Elle", "Brenda", "Phoebe", "Timothy", "Raschelle", "Lilly", "Burt", "Rina",
-            "Rodney", "Maris", "Jaron", "Wilf", "Harlan", "Audra", "Vincent", "Elwyn", "Drew", "Wynter", "Ora",
-            "Lissa", "Virgil", "Xavier", "Chad", "Ollie", "Leyton", "Karolyn", "Skye", "Roni", "Gladys", "Dinah",
-            "Penny", "August", "Osmund", "Whitaker", "Brande", "Cornell", "Phil", "Zara", "Kilie", "Gavin", "Coty",
-            "Randy", "Teri", "Keira", "Pru", "Clemency", "Kelcey", "Nevil", "Poppy", "Gareth", "Christabel", "Bastian",
-            "Wynonna", "Roselyn", "Goddard", "Collin", "Trace", "Neal", "Effie", "Denys", "Virginia", "Richard",
-            "Isiah", "Harrietta", "Gaylord", "Diamond", "Trudi", "Elaine", "Jemmy", "Gage", "Annabel", "Quincy", "Syd",
-            "Marianna", "Philomena", "Aubree", "Kathie", "Jacki", "Kelley", "Bess", "Cecil", "Maryvonne", "Kassidy",
-            "Anselm", "Dona", "Darby", "Jamison", "Daryl", "Darell", "Teal", "Lennie", "Bartholomew", "Katie",
-            "Maybelline", "Kimball", "Elvis", "Les", "Flick", "Harley", "Beth", "Bidelia", "Montague", "Helen", "Ozzy",
-            "Stef", "Debra", "Maxene", "Stefanie", "Russ", "Avril", "Johnathan", "Orson", "Chelsey", "Josephine",
-            "Deshaun", "Wendell", "Lula", "Ferdinanda", "Greg", "Brad", "Kynaston", "Dena", "Russel", "Robertina",
-            "Misti", "Leon", "Anjelica", "Bryana", "Myles", "Judi", "Curtis", "Davin", "Kristia", "Chrysanta",
-            "Hayleigh", "Hector", "Osbert", "Eustace", "Cary", "Tansy", "Cayley", "Maryann", "Alissa", "Ike",
-            "Tranter", "Reina", "Alwilda", "Sidony", "Columbine", "Astra", "Jillie", "Stephania", "Jonah", "Kennedy",
-            "Ferdinand", "Allegria", "Donella", "Kelleigh", "Darian", "Eldreda", "Jayden", "Herbie", "Jake", "Winston",
-            "Vi", "Annie", "Cherice", "Hugo", "Tricia", "Haydee", "Cassarah", "Darden", "Mallory", "Alton", "Hadley",
-            "Romayne", "Lacey", "Ern", "Alayna", "Cecilia", "Seward", "Tilly", "Edgar", "Concordia", "Ibbie", "Dahlia",
-            "Oswin", "Stu", "Brett", "Maralyn", "Kristeen", "Dotty", "Robyn", "Nessa", "Tresha", "Guinevere",
-            "Emerson", "Haze", "Lyn", "Henderson", "Lexa", "Jaylen", "Gail", "Lizette", "Tiara", "Robbie", "Destiny",
-            "Alice", "Livia", "Rosy", "Leah", "Jan", "Zach", "Vita", "Gia", "Micheal", "Rowina", "Alysha", "Bobbi",
-            "Delores", "Osmond", "Karaugh", "Wilbur", "Kasandra", "Renae", "Kaety", "Dora", "Gaye", "Amaryllis",
-            "Katelyn", "Dacre", "Prudence", "Ebony", "Camron", "Jerrold", "Vivyan", "Randall", "Donna", "Misty",
-            "Damon", "Selby", "Esmund", "Rian", "Garry", "Julius", "Raelene", "Clement", "Dom", "Tibby", "Moss",
-            "Millicent", "Gwendoline", "Berry", "Ashleigh", "Lilac", "Quin", "Vere", "Creighton", "Harriet", "Malvina",
-            "Lianne", "Pearle", "Kizzie", "Kara", "Petula", "Jeanie", "Maria", "Pacey", "Victoria", "Huey", "Toni",
-            "Rose", "Wallis", "Diggory", "Josiah", "Delma", "Keysha", "Channing", "Prue", "Lee", "Ryan", "Sidney",
-            "Valerie", "Clancy", "Ezra", "Gilbert", "Clare", "Laz", "Crofton", "Mike", "Annabella", "Tara", "Eldred",
-            "Arthur", "Jaylon", "Peronel", "Paden", "Dot", "Marian", "Amyas", "Alexus", "Esmond", "Abbie", "Stanley",
-            "Brittani", "Vickie", "Errol", "Kimberlee", "Uland", "Ebenezer", "Howie", "Eveline", "Andrea", "Trish",
-            "Hopkin", "Bryanna", "Temperance", "Valarie", "Femie", "Alix", "Terrell", "Lewin", "Lorrin", "Happy",
-            "Micah", "Rachyl", "Sloan", "Gertrude", "Elizabeth", "Dorris", "Andra", "Bram", "Gary", "Jeannine",
-            "Maurene", "Irene", "Yolonda", "Jonty", "Coleen", "Cecelia", "Chantal", "Stuart", "Caris", "Ros",
-            "Kaleigh", "Mirabelle", "Kolby", "Primrose", "Susannah", "Ginny", "Jinny", "Dolly", "Lettice", "Sonny",
-            "Melva", "Ernest", "Garret", "Reagan", "Trenton", "Gallagher", "Edwin", "Nikolas", "Corrie", "Lynette",
-            "Ettie", "Sly", "Debbi", "Eudora", "Brittney", "Tacey", "Marius", "Anima", "Gordon", "Olivia", "Kortney",
-            "Shantel", "Kolleen", "Nevaeh", "Buck", "Sera", "Liliana", "Aric", "Kalyn", "Mick", "Libby", "Ingram",
-            "Alexandria", "Darleen", "Jacklyn", "Hughie", "Tyler", "Aida", "Ronda", "Deemer", "Taryn", "Laureen",
-            "Samantha", "Dave", "Hardy", "Baldric", "Montgomery", "Gus", "Ellis", "Titania", "Luke", "Chase", "Haidee",
-            "Mayra", "Isabell", "Trinity", "Milo", "Abigail", "Tacita", "Meg", "Hervey", "Natasha", "Sadie", "Holden",
-            "Dee", "Mansel", "Perry", "Randi", "Frederica", "Georgina", "Kolour", "Debbie", "Seraphina", "Elspet",
-            "Julyan", "Raven", "Zavia", "Jarvis", "Jaymes", "Grover", "Cairo", "Alea", "Jordon", "Braxton", "Donny",
-            "Rhoda", "Tonya", "Bee", "Alyssia", "Ashlyn", "Reanna", "Lonny", "Arlene", "Deb", "Jane", "Nikole",
-            "Bettina", "Harrison", "Tamzen", "Arielle", "Adelaide", "Faith", "Bridie", "Wilburn", "Fern", "Nan",
-            "Shaw", "Zeke", "Alan", "Dene", "Gina", "Alexa", "Bailey", "Sal", "Tammy", "Maximillian", "America",
-            "Sylvana", "Fitz", "Mo", "Marissa", "Cass", "Eldon", "Wilfrid", "Tel", "Joann", "Kendra", "Tolly",
-            "Leanne", "Ferdie", "Haven", "Lucas", "Marlee", "Cyrilla", "Red", "Phoenix", "Jazmin", "Carin", "Gena",
-            "Lashonda", "Tucker", "Genette", "Kizzy", "Winifred", "Melody", "Keely", "Kaylyn", "Radcliff", "Lettie",
-            "Foster", "Lyndsey", "Nicholas", "Farley", "Louisa", "Dana", "Dortha", "Francine", "Doran", "Bonita",
-            "Hal", "Sawyer", "Reginald", "Aislin", "Nathan", "Baylee", "Abilene", "Ladonna", "Maurine", "Shelly",
-            "Deandre", "Jasmin", "Roderic", "Tiffany", "Amanda", "Verity", "Wilford", "Gayelord", "Whitney", "Demelza",
-            "Kenton", "Alberta", "Kyra", "Tabitha", "Sampson", "Korey", "Lillian", "Edison", "Clayton", "Steph",
-            "Maya", "Dusty", "Jim", "Ronny", "Adrianne", "Bernard", "Harris", "Kiley", "Alexander", "Kisha", "Ethalyn",
-            "Patience", "Briony", "Indigo", "Aureole", "Makenzie", "Molly", "Sherilyn", "Barry", "Laverne", "Hunter",
-            "Rocky", "Tyreek", "Madalyn", "Phyliss", "Chet", "Beatrice", "Faye", "Lavina", "Madelyn", "Tracey",
-            "Gyles", "Patti", "Carlyn", "Stephanie", "Jackalyn", "Larrie", "Kimmy", "Isolda", "Emelina", "Lis",
-            "Zillah", "Cody", "Sheard", "Rufus", "Paget", "Mae", "Rexanne", "Luvinia", "Tamsen", "Rosanna", "Greig",
-            "Stacia", "Mabelle", "Quianna", "Lotus", "Delice", "Bradford", "Angus", "Cosmo", "Earlene", "Adrian",
-            "Arlie", "Noelle", "Sabella", "Isa", "Adelle", "Innocent", "Kirby", "Trixie", "Kenelm", "Nelda", "Melia",
-            "Kendal", "Dorinda", "Placid", "Linette", "Kam", "Sherisse", "Evan", "Ewart", "Janice", "Linton",
-            "Jacaline", "Charissa", "Douglas", "Aileen", "Kemp", "Oli", "Amethyst", "Rosie", "Nigella", "Sherill",
-            "Anderson", "Alanna", "Eric", "Claudia", "Jennifer", "Boniface", "Harriet", "Vernon", "Lucy", "Shawnee",
-            "Gerard", "Cecily", "Romey", "Randall", "Wade", "Lux", "Dawson", "Gregg", "Kade", "Roxanne", "Melinda",
-            "Rolland", "Rowanne", "Fannie", "Isidore", "Melia", "Harvie", "Salal", "Eleonor", "Jacquette", "Lavone",
-            "Shanika", "Tarquin", "Janet", "Josslyn", "Maegan", "Augusta", "Aubree", "Francene", "Martie", "Marisa",
-            "Tyreek", "Tatianna", "Caleb", "Sheridan", "Nellie", "Barbara", "Wat", "Jayla", "Esmaralda", "Graeme",
-            "Lavena", "Jemima", "Nikolas", "Triston", "Portia", "Kyla", "Marcus", "Raeburn", "Jamison", "Earl", "Wren",
-            "Leighton", "Lagina", "Lucasta", "Dina", "Amaranta", "Jessika", "Claud", "Bernard", "Winifred", "Ebba",
-            "Sammi", "Gall", "Chloe", "Ottoline", "Herbert", "Janice", "Gareth", "Channing", "Caleigh", "Kailee",
-            "Ralphie", "Tamzen", "Quincy", "Beaumont", "Albert", "Jadyn", "Violet", "Luanna", "Moriah", "Humbert",
-            "Jed", "Leona", "Hale", "Mitch", "Marlin", "Nivek", "Darwin", "Dirk", "Liliana", "Meadow", "Bernadine",
-            "Jorie", "Peyton", "Astra", "Roscoe", "Gina", "Lovell", "Jewel", "Romayne", "Rosy", "Imogene",
-            "Margaretta", "Lorinda", "Hopkin", "Bobby", "Flossie", "Bennie", "Horatio", "Jonah", "Lyn", "Deana",
-            "Juliana", "Blanch", "Wright", "Kendal", "Woodrow", "Tania", "Austyn", "Val", "Mona", "Charla", "Rudyard",
-            "Pamela", "Raven", "Zena", "Nicola", "Kaelea", "Conor", "Virgil", "Sonnie", "Goodwin", "Christianne",
-            "Linford", "Myron", "Denton", "Charita", "Brody", "Ginnie", "Harrison", "Jeanine", "Quin", "Isolda",
-            "Zoie", "Pearce", "Margie", "Larrie", "Angelina", "Marcia", "Jessamine", "Delilah", "Dick", "Luana",
-            "Delicia", "Lake", "Luvenia", "Vaughan", "Concordia", "Gayelord", "Cheyenne", "Felix", "Dorris", "Pen",
-            "Kristeen", "Parris", "Everitt", "Josephina", "Amy", "Tommie", "Adrian", "April", "Rosaline", "Zachery",
-            "Trace", "Phoebe", "Jenelle", "Kameron", "Katharine", "Media", "Colton", "Tad", "Quianna", "Kerenza",
-            "Greta", "Luvinia", "Pete", "Tonya", "Beckah", "Barbra", "Jon", "Tetty", "Corey", "Sylvana", "Kizzy",
-            "Korey", "Trey", "Haydee", "Penny", "Mandy", "Panda", "Coline", "Ramsey", "Sukie", "Annabel", "Sarina",
-            "Corbin", "Suzanna", "Rob", "Duana", "Shell", "Jason", "Eddy", "Rube", "Roseann", "Celia", "Brianne",
-            "Nerissa", "Jera", "Humphry", "Ashlynn", "Terrence", "Philippina", "Coreen", "Kolour", "Indiana", "Paget",
-            "Marlyn", "Hester", "Isbel", "Ocean", "Harris", "Leslie", "Vere", "Monroe", "Isabelle", "Bertie", "Clitus",
-            "Dave", "Alethea", "Lessie", "Louiza", "Madlyn", "Garland", "Wolf", "Lalo", "Donny", "Amabel", "Tianna",
-            "Louie", "Susie", "Mackenzie", "Renie", "Tess", "Marmaduke", "Gwendolen", "Bettina", "Beatrix", "Esmund",
-            "Minnie", "Carlie", "Barnabas", "Ruthie", "Honour", "Haylie", "Xavior", "Freddie", "Ericka", "Aretha",
-            "Edie", "Madelina", "Anson", "Tabby", "Derrick", "Jocosa", "Deirdre", "Aislin", "Chastity", "Abigail",
-            "Wynonna", "Zo", "Eldon", "Krystine", "Ghislaine", "Zavia", "Nolene", "Marigold", "Kelley", "Sylvester",
-            "Odell", "George", "Laurene", "Franklyn", "Clarice", "Mo", "Dustin", "Debbi", "Lina", "Tony", "Acacia",
-            "Hettie", "Natalee", "Marcie", "Brittany", "Elnora", "Rachel", "Dawn", "Basil", "Christal", "Anjelica",
-            "Fran", "Tawny", "Delroy", "Tameka", "Lillie", "Ceara", "Deanna", "Deshaun", "Ken", "Bradford", "Justina",
-            "Merle", "Draven", "Gretta", "Harriette", "Webster", "Nathaniel", "Anemone", "Coleen", "Ruth", "Chryssa",
-            "Hortensia", "Saffie", "Deonne", "Leopold", "Harlan", "Lea", "Eppie", "Lucinda", "Tilda", "Fanny", "Titty",
-            "Lockie", "Jepson", "Sherisse", "Maralyn", "Ethel", "Sly", "Ebenezer", "Canute", "Ella", "Freeman",
-            "Reuben", "Olivette", "Nona", "Rik", "Amice", "Kristine", "Kathie", "Jayne", "Jeri", "Mckenna", "Bertram",
-            "Kaylee", "Livia", "Gil", "Wallace", "Maryann", "Keeleigh", "Laurinda", "Doran", "Khloe", "Dakota",
-            "Yaron", "Kimberleigh", "Gytha", "Doris", "Marylyn", "Benton", "Linnette", "Esther", "Jakki", "Rowina",
-            "Marian", "Roselyn", "Norbert", "Maggie", "Caesar", "Phinehas", "Jerry", "Jasmine", "Antonette", "Miriam",
-            "Monna", "Maryvonne", "Jacquetta", "Bernetta", "Napier", "Annie", "Gladwin", "Sheldon", "Aric", "Elouise",
-            "Gawain", "Kristia", "Gabe", "Kyra", "Red", "Tod", "Dudley", "Lorraine", "Ryley", "Sabina", "Poppy",
-            "Leland", "Aileen", "Eglantine", "Alicia", "Jeni", "Addy", "Tiffany", "Geffrey", "Lavina", "Collin",
-            "Clover", "Vin", "Jerome", "Doug", "Vincent", "Florence", "Scarlet", "Celeste", "Desdemona", "Tiphanie",
-            "Kassandra", "Ashton", "Madison", "Art", "Magdalene", "Iona", "Josepha", "Anise", "Ferne", "Derek",
-            "Huffie", "Qiana", "Ysabel", "Tami", "Shannah", "Xavier", "Willard", "Winthrop", "Vickie", "Maura",
-            "Placid", "Tiara", "Reggie", "Elissa", "Isa", "Chrysanta", "Jeff", "Bessie", "Terri", "Amilia", "Brett",
-            "Daniella", "Damion", "Carolina", "Maximillian", "Travers", "Benjamin", "Oprah", "Darcy", "Yolanda",
-            "Nicolina", "Crofton", "Jarrett", "Kaitlin", "Shauna", "Keren", "Bevis", "Kalysta", "Sharron", "Alyssa",
-            "Blythe", "Zelma", "Caelie", "Norwood", "Billie", "Patrick", "Gary", "Cambria", "Tylar", "Mason", "Helen",
-            "Melyssa", "Gene", "Gilberta", "Carter", "Herbie", "Harmonie", "Leola", "Eugenia", "Clint", "Pauletta",
-            "Edwyna", "Georgina", "Teal", "Harper", "Izzy", "Dillon", "Kezia", "Evangeline", "Colene", "Madelaine",
-            "Zilla", "Rudy", "Dottie", "Caris", "Morton", "Marge", "Tacey", "Parker", "Troy", "Liza", "Lewin",
-            "Tracie", "Justine", "Dallas", "Linden", "Ray", "Loretta", "Teri", "Elvis", "Diane", "Julianna", "Manfred",
-            "Denise", "Eireen", "Ann", "Kenith", "Linwood", "Kathlyn", "Bernice", "Shelley", "Oswald", "Amedeus",
-            "Homer", "Tanzi", "Ted", "Ralphina", "Hyacinth", "Lotus", "Matthias", "Arlette", "Clark", "Cecil",
-            "Elspeth", "Alvena", "Noah", "Millard", "Brenden", "Cole", "Philipa", "Nina", "Thelma", "Iantha", "Reid",
-            "Jefferson", "Meg", "Elsie", "Shirlee", "Nathan", "Nancy", "Simona", "Racheal", "Carin", "Emory", "Delice",
-            "Kristi", "Karaugh", "Kaety", "Tilly", "Em", "Alanis", "Darrin", "Jerrie", "Hollis", "Cary", "Marly",
-            "Carita", "Jody", "Farley", "Hervey", "Rosalin", "Cuthbert", "Stewart", "Jodene", "Caileigh", "Briscoe",
-            "Dolores", "Sheree", "Eustace", "Nigel", "Detta", "Barret", "Rowland", "Kenny", "Githa", "Zoey", "Adela",
-            "Petronella", "Opal", "Coleman", "Niles", "Cyril", "Dona", "Alberic", "Allannah", "Jules", "Avalon",
-            "Hadley", "Thomas", "Renita", "Calanthe", "Heron", "Shawnda", "Chet", "Malina", "Manny", "Rina", "Frieda",
-            "Eveleen", "Deshawn", "Amos", "Raelene", "Paige", "Molly", "Nannie", "Ileen", "Brendon", "Milford",
-            "Unice", "Rebeccah", "Caedmon", "Gae", "Doreen", "Vivian", "Louis", "Raphael", "Vergil", "Lise", "Glenn",
-            "Karyn", "Terance", "Reina", "Jake", "Gordon", "Wisdom", "Isiah", "Gervase", "Fern", "Marylou", "Roddy",
-            "Justy", "Derick", "Shantelle", "Adam", "Chantel", "Madoline", "Emmerson", "Lexie", "Mickey", "Stephen",
-            "Dane", "Stacee", "Elwin", "Tracey", "Alexandra", "Ricky", "Ian", "Kasey", "Rita", "Alanna", "Georgene",
-            "Deon", "Zavier", "Ophelia", "Deforest", "Lowell", "Zubin", "Hardy", "Osmund", "Tabatha", "Debby",
-            "Katlyn", "Tallulah", "Priscilla", "Braden", "Wil", "Keziah", "Jen", "Aggie", "Korbin", "Lemoine",
-            "Barnaby", "Tranter", "Goldie", "Roderick", "Trina", "Emery", "Pris", "Sidony", "Adelle", "Tate", "Wilf",
-            "Zola", "Brande", "Chris", "Calanthia", "Lilly", "Kaycee", "Lashonda", "Jasmin", "Elijah", "Shantel",
-            "Simon", "Rosalind", "Jarod", "Kaylie", "Corrine", "Joselyn", "Archibald", "Mariabella", "Winton",
-            "Merlin", "Chad", "Ursula", "Kristopher", "Hewie", "Adrianna", "Lyndsay", "Jasmyn", "Tim", "Evette",
-            "Margaret", "Samson", "Bronte", "Terence", "Leila", "Candice", "Tori", "Jamey", "Coriander", "Conrad",
-            "Floyd", "Karen", "Lorin", "Maximilian", "Cairo", "Emily", "Yasmin", "Karolyn", "Bryan", "Lanny",
-            "Kimberly", "Rick", "Chaz", "Krystle", "Lyric", "Laura", "Garrick", "Flip", "Monty", "Brendan",
-            "Ermintrude", "Rayner", "Merla", "Titus", "Marva", "Patricia", "Leone", "Tracy", "Jaqueline", "Hallam",
-            "Delores", "Cressida", "Carlyle", "Leann", "Kelcey", "Laurence", "Ryan", "Reynold", "Mark", "Collyn",
-            "Audie", "Sammy", "Ellery", "Sallie", "Pamelia", "Adolph", "Lydia", "Titania", "Ron", "Bridger", "Aline",
-            "Read", "Kelleigh", "Weldon", "Irving", "Garey", "Diggory", "Evander", "Kylee", "Deidre", "Ormond",
-            "Laurine", "Reannon", "Arline", "Pat"
-
-    };
-
-    public static String[] jargon = { "wireless", "signal", "network", "3G", "plan", "touch-screen",
-            "customer-service", "reachability", "voice-command", "shortcut-menu", "customization", "platform", "speed",
-            "voice-clarity", "voicemail-service" };
-
-    public static String[] vendors = { "at&t", "verizon", "t-mobile", "sprint", "motorola", "samsung", "iphone" };
-
-    public static String[] org_list = { "Latsonity", "ganjalax", "Zuncan", "Lexitechno", "Hot-tech", "subtam",
-            "Coneflex", "Ganjatax", "physcane", "Tranzap", "Qvohouse", "Zununoing", "jaydax", "Keytech", "goldendexon",
-            "Villa-tech", "Trustbam", "Newcom", "Voltlane", "Ontohothex", "Ranhotfan", "Alphadax", "Transhigh",
-            "kin-ron", "Doublezone", "Solophase", "Vivaace", "silfind", "Basecone", "sonstreet", "Freshfix",
-            "Techitechi", "Kanelectrics", "linedexon", "Goldcity", "Newfase", "Technohow", "Zimcone", "Salthex",
-            "U-ron", "Solfix", "whitestreet", "Xx-technology", "Hexviafind", "over-it", "Strongtone", "Tripplelane",
-            "geomedia", "Scotcity", "Inchex", "Vaiatech", "Striptaxon", "Hatcom", "tresline", "Sanjodax", "freshdox",
-            "Sumlane", "Quadlane", "Newphase", "overtech", "Voltbam", "Icerunin", "Fixdintex", "Hexsanhex", "Statcode",
-            "Greencare", "U-electrics", "Zamcorporation", "Ontotanin", "Tanzimcare", "Groovetex", "Ganjastrip",
-            "Redelectronics", "Dandamace", "Whitemedia", "strongex", "Streettax", "highfax", "Mathtech", "Xx-drill",
-            "Sublamdox", "Unijobam", "Rungozoom", "Fixelectrics", "Villa-dox", "Ransaofan", "Plexlane", "itlab",
-            "Lexicone", "Fax-fax", "Viatechi", "Inchdox", "Kongreen", "Doncare", "Y-geohex", "Opeelectronics",
-            "Medflex", "Dancode", "Roundhex", "Labzatron", "Newhotplus", "Sancone", "Ronholdings", "Quoline",
-            "zoomplus", "Fix-touch", "Codetechno", "Tanzumbam", "Indiex", "Canline" };
-}
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/DataGeneratorDriver.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/DataGeneratorDriver.java
deleted file mode 100644
index b2ba88e..0000000
--- a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/DataGeneratorDriver.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package edu.uci.ics.asterix.tools.external.data;
-
-import java.util.Iterator;
-
-import edu.uci.ics.asterix.tools.external.data.DataGenerator.InitializationInfo;
-import edu.uci.ics.asterix.tools.external.data.DataGenerator.TweetMessage;
-
-public class DataGeneratorDriver {
-
-    public static void main(String[] args) {
-
-        DataGenerator.initialize(new InitializationInfo());
-        Iterator<TweetMessage> tweetIterator = DataGenerator.getTwitterMessageIterator();
-        while (tweetIterator.hasNext()) {
-            System.out.println(tweetIterator.next().toString());
-        }
-    }
-}
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/GULongIDGenerator.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/GULongIDGenerator.java
index 59c133c..61c4345 100644
--- a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/GULongIDGenerator.java
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/GULongIDGenerator.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package edu.uci.ics.asterix.tools.external.data;
 
 import java.nio.ByteBuffer;
@@ -5,28 +19,28 @@
 
 public class GULongIDGenerator {
 
-    private final int partition;
-    private final long baseValue;
-    private final AtomicLong nextValue;
+	private final int partition;
+	private final long baseValue;
+	private final AtomicLong nextValue;
 
-    public GULongIDGenerator(int partition, byte seed) {
-        this.partition = partition;
-        ByteBuffer buffer = ByteBuffer.allocate(8);
-        buffer.put(seed);
-        buffer.put((byte) partition);
-        buffer.putInt(0);
-        buffer.putShort((short) 0);
-        buffer.flip();
-        this.baseValue = new Long(buffer.getLong());
-        this.nextValue = new AtomicLong(baseValue);
-    }
+	public GULongIDGenerator(int partition, byte seed) {
+		this.partition = partition;
+		ByteBuffer buffer = ByteBuffer.allocate(8);
+		buffer.put(seed);
+		buffer.put((byte) partition);
+		buffer.putInt(0);
+		buffer.putShort((short) 0);
+		buffer.flip();
+		this.baseValue = new Long(buffer.getLong());
+		this.nextValue = new AtomicLong(baseValue);
+	}
 
-    public long getNextULong() {
-        return nextValue.incrementAndGet();
-    }
-    
-    public int getPartition(){
-        return partition;
-    }
- 
+	public long getNextULong() {
+		return nextValue.incrementAndGet();
+	}
+
+	public int getPartition() {
+		return partition;
+	}
+
 }
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/GenericSocketFeedAdapter.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/GenericSocketFeedAdapter.java
index a536076..31c50a7 100644
--- a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/GenericSocketFeedAdapter.java
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/GenericSocketFeedAdapter.java
@@ -73,6 +73,9 @@
                 socket = serverSocket.accept();
                 inputStream = socket.getInputStream();
             } catch (IOException e) {
+            	if (LOGGER.isLoggable(Level.SEVERE)) {
+                    LOGGER.severe("Unable to create input stream required for feed ingestion");
+                }
                 e.printStackTrace();
             }
             return inputStream;
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/SyntheticTwitterFeedAdapter.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/SyntheticTwitterFeedAdapter.java
deleted file mode 100644
index d26e764..0000000
--- a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/SyntheticTwitterFeedAdapter.java
+++ /dev/null
@@ -1,214 +0,0 @@
-package edu.uci.ics.asterix.tools.external.data;
-
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Logger;
-
-import edu.uci.ics.asterix.common.exceptions.AsterixException;
-import edu.uci.ics.asterix.dataflow.data.nontagged.serde.ARecordSerializerDeserializer;
-import edu.uci.ics.asterix.external.dataset.adapter.IPullBasedFeedClient;
-import edu.uci.ics.asterix.external.dataset.adapter.PullBasedAdapter;
-import edu.uci.ics.asterix.external.dataset.adapter.PullBasedFeedClient;
-import edu.uci.ics.asterix.om.base.AMutableDateTime;
-import edu.uci.ics.asterix.om.base.AMutableInt32;
-import edu.uci.ics.asterix.om.base.AMutablePoint;
-import edu.uci.ics.asterix.om.base.AMutableRecord;
-import edu.uci.ics.asterix.om.base.AMutableString;
-import edu.uci.ics.asterix.om.base.AMutableUnorderedList;
-import edu.uci.ics.asterix.om.base.IAObject;
-import edu.uci.ics.asterix.om.types.ARecordType;
-import edu.uci.ics.asterix.om.types.AUnorderedListType;
-import edu.uci.ics.asterix.om.types.BuiltinType;
-import edu.uci.ics.asterix.tools.external.data.DataGenerator.InitializationInfo;
-import edu.uci.ics.asterix.tools.external.data.DataGenerator.Message;
-import edu.uci.ics.asterix.tools.external.data.DataGenerator.TweetMessage;
-import edu.uci.ics.asterix.tools.external.data.DataGenerator.TweetMessageIterator;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-
-/**
- * TPS can be configured between 1 and 20,000
- * 
- * @author ramang
- */
-public class SyntheticTwitterFeedAdapter extends PullBasedAdapter {
-
-    private static final long serialVersionUID = 1L;
-
-    private static final Logger LOGGER = Logger.getLogger(SyntheticTwitterFeedAdapter.class.getName());
-
-    private Map<String, String> configuration;
-
-    public SyntheticTwitterFeedAdapter(Map<String, String> configuration, ARecordType outputType,
-            IHyracksTaskContext ctx) throws AsterixException {
-        super(configuration, ctx);
-        this.configuration = configuration;
-        this.adapterOutputType = outputType;
-    }
-
-    @Override
-    public IPullBasedFeedClient getFeedClient(int partition) throws Exception {
-        return new SyntheticTwitterFeedClient(configuration, adapterOutputType, partition);
-    }
-
-    private static class SyntheticTwitterFeedClient extends PullBasedFeedClient implements IPullBasedFeedClient {
-
-        private static final Logger LOGGER = Logger.getLogger(SyntheticTwitterFeedClient.class.getName());
-
-        public static final String KEY_DURATION = "duration";
-        public static final String KEY_TPS = "tps";
-        public static final String KEY_EXCEPTION_PERIOD = "exception-period";
-
-        private int duration;
-        private long tweetInterval;
-        private int numTweetsBeforeDelay;
-        private TweetMessageIterator tweetIterator = null;
-        private long exeptionInterval;
-
-        private IAObject[] mutableFields;
-        private ARecordType outputRecordType;
-        private int partition;
-        private int tweetCount = 0;
-        private int tweetCountBeforeException = 0;
-        private int exceptionPeriod = -1;
-
-        public SyntheticTwitterFeedClient(Map<String, String> configuration, ARecordType outputRecordType, int partition)
-                throws AsterixException {
-            this.outputRecordType = outputRecordType;
-            String value = (String) configuration.get(KEY_DURATION);
-            duration = value != null ? Integer.parseInt(value) : 60;
-            initializeTweetRate((String) configuration.get(KEY_TPS));
-            value = (String) configuration.get(KEY_EXCEPTION_PERIOD);
-            if (value != null) {
-                exceptionPeriod = Integer.parseInt(value);
-            }
-
-            InitializationInfo info = new InitializationInfo();
-            info.timeDurationInSecs = duration;
-            DataGenerator.initialize(info);
-            tweetIterator = new TweetMessageIterator(duration);
-            initialize();
-        }
-
-        private void initializeTweetRate(String tps) {
-            numTweetsBeforeDelay = 0;
-            if (tps == null) {
-                tweetInterval = 0;
-            } else {
-                int val = Integer.parseInt(tps);
-                double interval = new Double(((double) 1000 / val));
-                if (interval > 1) {
-                    tweetInterval = (long) interval;
-                    numTweetsBeforeDelay = 1;
-                } else {
-                    tweetInterval = 1;
-                    Double numTweets = new Double(1 / interval);
-                    if (numTweets.intValue() != numTweets) {
-                        tweetInterval = 10;
-                        numTweetsBeforeDelay = (new Double(10 * numTweets * 1.4)).intValue();
-                    } else {
-                        numTweetsBeforeDelay = new Double((numTweets * 1.4)).intValue();
-                    }
-                }
-            }
-
-        }
-
-        private void writeTweet(TweetMessage next) {
-
-            // tweet id
-            LOGGER.info("Generating next tweet");
-            ((AMutableString) mutableFields[0]).setValue(next.getTweetid());
-            mutableRecord.setValueAtPos(0, mutableFields[0]);
-
-            // user
-            AMutableRecord userRecord = ((AMutableRecord) mutableFields[1]);
-            ((AMutableString) userRecord.getValueByPos(0)).setValue(next.getUser().getScreenName());
-            ((AMutableString) userRecord.getValueByPos(1)).setValue("en");
-            ((AMutableInt32) userRecord.getValueByPos(2)).setValue(next.getUser().getFriendsCount());
-            ((AMutableInt32) userRecord.getValueByPos(3)).setValue(next.getUser().getStatusesCount());
-            ((AMutableString) userRecord.getValueByPos(4)).setValue(next.getUser().getName());
-            ((AMutableInt32) userRecord.getValueByPos(5)).setValue(next.getUser().getFollowersCount());
-            mutableRecord.setValueAtPos(1, userRecord);
-
-            // location
-            ((AMutablePoint) mutableFields[2]).setValue(next.getSenderLocation().getLatitude(), next
-                    .getSenderLocation().getLongitude());
-            mutableRecord.setValueAtPos(2, mutableFields[2]);
-
-            // time
-            ((AMutableDateTime) mutableFields[3]).setValue(next.getSendTime().getChrononTime());
-            mutableRecord.setValueAtPos(3, mutableFields[3]);
-
-            // referred topics
-            ((AMutableUnorderedList) mutableFields[4]).clear();
-            List<String> referredTopics = next.getReferredTopics();
-            for (String topic : referredTopics) {
-                ((AMutableUnorderedList) mutableFields[4]).add(new AMutableString(topic));
-            }
-            mutableRecord.setValueAtPos(4, mutableFields[4]);
-
-            // text
-            Message m = next.getMessageText();
-            char[] content = m.getMessage();
-            String tweetText = new String(content, 0, m.getLength());
-            ((AMutableString) mutableFields[5]).setValue(tweetText);
-            mutableRecord.setValueAtPos(5, mutableFields[5]);
-            LOGGER.info(tweetText);
-
-        }
-
-        @Override
-        public void resetOnFailure(Exception e) throws AsterixException {
-            // TODO Auto-generated method stub
-
-        }
-
-        @Override
-        public boolean alter(Map<String, String> configuration) {
-            // TODO Auto-generated method stub
-            return false;
-        }
-
-        @Override
-        public InflowState setNextRecord() throws Exception {
-            boolean moreData = tweetIterator.hasNext();
-            if (!moreData) {
-                return InflowState.NO_MORE_DATA;
-            }
-            writeTweet(tweetIterator.next());
-            if (tweetInterval != 0) {
-                tweetCount++;
-                if (tweetCount == numTweetsBeforeDelay) {
-                    Thread.sleep(tweetInterval);
-                    tweetCount = 0;
-                }
-            }
-            tweetCountBeforeException++;
-
-            if (tweetCountBeforeException == exceptionPeriod) {
-                tweetCountBeforeException = 0;
-                throw new AsterixException("Delibrate exception");
-            }
-            return InflowState.DATA_AVAILABLE;
-        }
-
-        private void initialize() throws AsterixException {
-            ARecordType userRecordType = (ARecordType) outputRecordType.getFieldTypes()[1];
-            IAObject[] userMutableFields = new IAObject[] { new AMutableString(""), new AMutableString(""),
-                    new AMutableInt32(0), new AMutableInt32(0), new AMutableString(""), new AMutableInt32(0) };
-            AUnorderedListType unorderedListType = new AUnorderedListType(BuiltinType.ASTRING, "referred-topics");
-            mutableFields = new IAObject[] { new AMutableString(""),
-                    new AMutableRecord(userRecordType, userMutableFields), new AMutablePoint(0, 0),
-                    new AMutableDateTime(0), new AMutableUnorderedList(unorderedListType), new AMutableString("") };
-            recordSerDe = new ARecordSerializerDeserializer(outputRecordType);
-            mutableRecord = new AMutableRecord(outputRecordType, mutableFields);
-
-        }
-
-        @Override
-        public void stop() {
-            // TODO Auto-generated method stub
-
-        }
-    }
-}
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/SyntheticTwitterFeedAdapterFactory.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/SyntheticTwitterFeedAdapterFactory.java
deleted file mode 100644
index baa7a63..0000000
--- a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/SyntheticTwitterFeedAdapterFactory.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
-x * Copyright 2009-2012 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.tools.external.data;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-
-import edu.uci.ics.asterix.common.exceptions.AsterixException;
-import edu.uci.ics.asterix.metadata.MetadataManager;
-import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
-import edu.uci.ics.asterix.metadata.entities.Dataset;
-import edu.uci.ics.asterix.metadata.entities.InternalDatasetDetails;
-import edu.uci.ics.asterix.metadata.entities.NodeGroup;
-import edu.uci.ics.asterix.metadata.feeds.IDatasourceAdapter;
-import edu.uci.ics.asterix.metadata.feeds.ITypedAdapterFactory;
-import edu.uci.ics.asterix.om.types.ARecordType;
-import edu.uci.ics.asterix.om.types.AUnorderedListType;
-import edu.uci.ics.asterix.om.types.BuiltinType;
-import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-
-/**
- * Factory class for creating @see{RateControllerFileSystemBasedAdapter} The
- * adapter simulates a feed from the contents of a source file. The file can be
- * on the local file system or on HDFS. The feed ends when the content of the
- * source file has been ingested.
- */
-public class SyntheticTwitterFeedAdapterFactory implements ITypedAdapterFactory {
-
-    /**
-     * 
-     */
-    private static final long serialVersionUID = 1L;
-
-    private Map<String, String> configuration;
-
-    private static final String KEY_DATAVERSE_DATASET = "dataverse-dataset";
-
-    private static final ARecordType outputType = initOutputType();
-
-    @Override
-    public String getName() {
-        return "synthetic_twitter_feed";
-    }
-
-    @Override
-    public AdapterType getAdapterType() {
-        return AdapterType.TYPED;
-    }
-
-    @Override
-    public SupportedOperation getSupportedOperations() {
-        return SupportedOperation.READ;
-    }
-
-    @Override
-    public void configure(Map<String, String> configuration) throws Exception {
-        this.configuration = configuration;
-    }
-
-    @Override
-    public AlgebricksPartitionConstraint getPartitionConstraint() throws Exception {
-        String dvds = (String) configuration.get(KEY_DATAVERSE_DATASET);
-        String[] components = dvds.split(":");
-        String dataverse = components[0];
-        String dataset = components[1];
-        MetadataTransactionContext ctx = null;
-        NodeGroup ng = null;
-        try {
-            ctx = MetadataManager.INSTANCE.beginTransaction();
-            Dataset ds = MetadataManager.INSTANCE.getDataset(ctx, dataverse, dataset);
-            String nodegroupName = ((InternalDatasetDetails) ds.getDatasetDetails()).getNodeGroupName();
-            ng = MetadataManager.INSTANCE.getNodegroup(ctx, nodegroupName);
-            MetadataManager.INSTANCE.commitTransaction(ctx);
-        } catch (Exception e) {
-            MetadataManager.INSTANCE.abortTransaction(ctx);
-            throw e;
-        }
-        List<String> storageNodes = ng.getNodeNames();
-        Set<String> nodes = AsterixAppContextInfo.getInstance().getMetadataProperties().getNodeNames();
-        String ingestionLocation = null;
-        if (nodes.size() > storageNodes.size()) {
-            nodes.removeAll(storageNodes);
-        }
-        String[] nodesArray = nodes.toArray(new String[] {});
-        Random r = new Random();
-        ingestionLocation = nodesArray[r.nextInt(nodes.size())];
-        return new AlgebricksAbsolutePartitionConstraint(new String[] { ingestionLocation });
-    }
-
-    @Override
-    public IDatasourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws Exception {
-        return new SyntheticTwitterFeedAdapter(configuration, outputType, ctx);
-    }
-
-    @Override
-    public ARecordType getAdapterOutputType() {
-        return outputType;
-    }
-
-    private static ARecordType initOutputType() {
-        ARecordType outputType = null;
-        try {
-            String[] userFieldNames = new String[] { "screen-name", "lang", "friends_count", "statuses_count", "name",
-                    "followers_count" };
-
-            IAType[] userFieldTypes = new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.AINT32,
-                    BuiltinType.AINT32, BuiltinType.ASTRING, BuiltinType.AINT32 };
-            ARecordType userRecordType = new ARecordType("TwitterUserType", userFieldNames, userFieldTypes, false);
-
-            String[] fieldNames = new String[] { "tweetid", "user", "sender-location", "send-time", "referred-topics",
-                    "message-text" };
-
-            AUnorderedListType unorderedListType = new AUnorderedListType(BuiltinType.ASTRING, "referred-topics");
-            IAType[] fieldTypes = new IAType[] { BuiltinType.ASTRING, userRecordType, BuiltinType.APOINT,
-                    BuiltinType.ADATETIME, unorderedListType, BuiltinType.ASTRING };
-            outputType = new ARecordType("TweetMessageType", fieldNames, fieldTypes, false);
-
-        } catch (AsterixException e) {
-            throw new IllegalStateException("Unable to initialize output type");
-        }
-        return outputType;
-    }
-}
\ No newline at end of file
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/TweetGenerator.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/TweetGenerator.java
index 9d47de2..b92c3fd 100644
--- a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/TweetGenerator.java
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/TweetGenerator.java
@@ -1,213 +1,100 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package edu.uci.ics.asterix.tools.external.data;
 
 import java.io.IOException;
 import java.io.OutputStream;
-import java.util.List;
+import java.nio.ByteBuffer;
 import java.util.Map;
-import java.util.logging.Logger;
 
-import edu.uci.ics.asterix.common.exceptions.AsterixException;
-import edu.uci.ics.asterix.dataflow.data.nontagged.serde.ARecordSerializerDeserializer;
-import edu.uci.ics.asterix.external.dataset.adapter.IPullBasedFeedClient;
-import edu.uci.ics.asterix.external.dataset.adapter.PullBasedFeedClient;
-import edu.uci.ics.asterix.om.base.AMutableDateTime;
-import edu.uci.ics.asterix.om.base.AMutableInt32;
-import edu.uci.ics.asterix.om.base.AMutablePoint;
-import edu.uci.ics.asterix.om.base.AMutableRecord;
-import edu.uci.ics.asterix.om.base.AMutableString;
-import edu.uci.ics.asterix.om.base.AMutableUnorderedList;
-import edu.uci.ics.asterix.om.base.IAObject;
-import edu.uci.ics.asterix.om.types.ARecordType;
-import edu.uci.ics.asterix.om.types.AUnorderedListType;
-import edu.uci.ics.asterix.om.types.BuiltinType;
 import edu.uci.ics.asterix.tools.external.data.DataGenerator.InitializationInfo;
-import edu.uci.ics.asterix.tools.external.data.DataGenerator.Message;
 import edu.uci.ics.asterix.tools.external.data.DataGenerator.TweetMessage;
 import edu.uci.ics.asterix.tools.external.data.DataGenerator.TweetMessageIterator;
 
-public class TweetGenerator extends PullBasedFeedClient implements IPullBasedFeedClient {
-
-    private static final Logger LOGGER = Logger.getLogger(TweetGenerator.class.getName());
+public class TweetGenerator {
 
     public static final String KEY_DURATION = "duration";
     public static final String KEY_TPS = "tps";
-    public static final String KEY_EXCEPTION_PERIOD = "exception-period";
-    public static final String OUTPUT_FORMAT = "output-format";
+    public static final String KEY_MIN_TPS = "tps-min";
+    public static final String KEY_MAX_TPS = "tps-max";
+    public static final String KEY_TPUT_DURATION = "tput-duration";
+    public static final String KEY_GUID_SEED = "guid-seed";
 
+    public static final String OUTPUT_FORMAT = "output-format";
     public static final String OUTPUT_FORMAT_ARECORD = "arecord";
     public static final String OUTPUT_FORMAT_ADM_STRING = "adm-string";
 
     private int duration;
-    private long tweetInterval;
-    private int numTweetsBeforeDelay;
     private TweetMessageIterator tweetIterator = null;
-    private long exeptionInterval;
-
-    private IAObject[] mutableFields;
-    private ARecordType outputRecordType;
-    private int partition;
-    private int tweetCount = 0;
-    private int tweetCountBeforeException = 0;
-    private int exceptionPeriod = -1;
-    private boolean isOutputFormatRecord = false;
-    private byte[] EOL = "\n".getBytes();
+    private int frameTweetCount = 0;
+    private int numFlushedTweets = 0;
     private OutputStream os;
+    private DataGenerator dataGenerator = null;
+    private ByteBuffer outputBuffer = ByteBuffer.allocate(32 * 1024);
 
-    public TweetGenerator(Map<String, String> configuration, ARecordType outputRecordType, int partition, String format)
-            throws AsterixException {
-        this.outputRecordType = outputRecordType;
+    public TweetGenerator(Map<String, String> configuration, int partition, String format) throws Exception {
         String value = configuration.get(KEY_DURATION);
         duration = value != null ? Integer.parseInt(value) : 60;
-        initializeTweetRate(configuration.get(KEY_TPS));
-        value = configuration.get(KEY_EXCEPTION_PERIOD);
-        if (value != null) {
-            exceptionPeriod = Integer.parseInt(value);
-        }
-        isOutputFormatRecord = format.equalsIgnoreCase(OUTPUT_FORMAT_ARECORD);
         InitializationInfo info = new InitializationInfo();
         info.timeDurationInSecs = duration;
-        DataGenerator.initialize(info);
-        tweetIterator = new TweetMessageIterator(duration);
-        initialize();
-    }
+        dataGenerator = new DataGenerator(info);
 
-    private void initializeTweetRate(String tps) {
-        numTweetsBeforeDelay = 0;
-        if (tps == null) {
-            tweetInterval = 0;
-        } else {
-            int val = Integer.parseInt(tps);
-            double interval = new Double(((double) 1000 / val));
-            if (interval > 1) {
-                tweetInterval = (long) interval;
-                numTweetsBeforeDelay = 1;
-            } else {
-                tweetInterval = 1;
-                Double numTweets = new Double(1 / interval);
-                if (numTweets.intValue() != numTweets) {
-                    tweetInterval = 10;
-                    numTweetsBeforeDelay = (new Double(10 * numTweets * 1.4)).intValue();
-                } else {
-                    numTweetsBeforeDelay = new Double((numTweets * 1.4)).intValue();
-                }
-            }
-        }
-
+        String seedValue = configuration.get(KEY_GUID_SEED);
+        int seedInt = seedValue != null ? Integer.parseInt(seedValue) : 0;
+        tweetIterator = dataGenerator.new TweetMessageIterator(duration, partition, (byte) seedInt);
     }
 
     private void writeTweetString(TweetMessage next) throws IOException {
-        String tweet = next.toString();
-        os.write(tweet.getBytes());
-        os.write(EOL);
-        /*
-        if (LOGGER.isLoggable(Level.INFO)) {
-            LOGGER.info(tweet);
-        }*/
-    }
-
-    private void writeTweetRecord(TweetMessage next) {
-
-        //tweet id
-        ((AMutableString) mutableFields[0]).setValue(next.getTweetid());
-        mutableRecord.setValueAtPos(0, mutableFields[0]);
-
-        // user 
-        AMutableRecord userRecord = ((AMutableRecord) mutableFields[1]);
-        ((AMutableString) userRecord.getValueByPos(0)).setValue(next.getUser().getScreenName());
-        ((AMutableString) userRecord.getValueByPos(1)).setValue("en");
-        ((AMutableInt32) userRecord.getValueByPos(2)).setValue(next.getUser().getFriendsCount());
-        ((AMutableInt32) userRecord.getValueByPos(3)).setValue(next.getUser().getStatusesCount());
-        ((AMutableString) userRecord.getValueByPos(4)).setValue(next.getUser().getName());
-        ((AMutableInt32) userRecord.getValueByPos(5)).setValue(next.getUser().getFollowersCount());
-        mutableRecord.setValueAtPos(1, userRecord);
-
-        // location
-        ((AMutablePoint) mutableFields[2]).setValue(next.getSenderLocation().getLatitude(), next.getSenderLocation()
-                .getLongitude());
-        mutableRecord.setValueAtPos(2, mutableFields[2]);
-
-        // time
-        ((AMutableDateTime) mutableFields[3]).setValue(next.getSendTime().getChrononTime());
-        mutableRecord.setValueAtPos(3, mutableFields[3]);
-
-        // referred topics
-        ((AMutableUnorderedList) mutableFields[4]).clear();
-        List<String> referredTopics = next.getReferredTopics();
-        for (String topic : referredTopics) {
-            ((AMutableUnorderedList) mutableFields[4]).add(new AMutableString(topic));
-        }
-        mutableRecord.setValueAtPos(4, mutableFields[4]);
-
-        // text
-        Message m = next.getMessageText();
-        char[] content = m.getMessage();
-        String tweetText = new String(content, 0, m.getLength());
-        ((AMutableString) mutableFields[5]).setValue(tweetText);
-        mutableRecord.setValueAtPos(5, mutableFields[5]);
-        LOGGER.info(tweetText);
-
-    }
-
-    @Override
-    public void resetOnFailure(Exception e) throws AsterixException {
-        // TODO Auto-generated method stub
-
-    }
-
-    @Override
-    public boolean alter(Map<String, String> configuration) {
-        // TODO Auto-generated method stub
-        return false;
-    }
-
-    @Override
-    public InflowState setNextRecord() throws Exception {
-        boolean moreData = tweetIterator.hasNext();
-        if (!moreData) {
-            return InflowState.NO_MORE_DATA;
-        }
-        TweetMessage msg = tweetIterator.next();
-        if (isOutputFormatRecord) {
-            writeTweetRecord(msg);
+        String tweet = next.toString() + "\n";
+        byte[] b = tweet.getBytes();
+        if (outputBuffer.position() + b.length > outputBuffer.limit()) {
+            flush();
+            numFlushedTweets += frameTweetCount;
+            frameTweetCount = 0;
+            outputBuffer.put(b);
+            frameTweetCount++;
         } else {
-            writeTweetString(msg);
+            outputBuffer.put(b);
+            frameTweetCount++;
         }
-        if (tweetInterval != 0) {
-            tweetCount++;
-            if (tweetCount == numTweetsBeforeDelay) {
-                Thread.sleep(tweetInterval);
-                tweetCount = 0;
+    }
+
+    public int getNumFlushedTweets() {
+        return numFlushedTweets;
+    }
+
+    private void flush() throws IOException {
+        outputBuffer.flip();
+        os.write(outputBuffer.array(), 0, outputBuffer.limit());
+        outputBuffer.position(0);
+        outputBuffer.limit(32 * 1024);
+    }
+
+    public boolean setNextRecordBatch(int numTweetsInBatch) throws Exception {
+        int count = 0;
+        if (tweetIterator.hasNext()) {
+            while (count < numTweetsInBatch) {
+                writeTweetString(tweetIterator.next());
+                count++;
             }
+            return true;
         }
-        tweetCountBeforeException++;
-
-        if (tweetCountBeforeException == exceptionPeriod) {
-            tweetCountBeforeException = 0;
-            throw new AsterixException("Delibrate exception");
-        }
-        return InflowState.DATA_AVAILABLE;
-    }
-
-    private void initialize() throws AsterixException {
-        ARecordType userRecordType = (ARecordType) outputRecordType.getFieldTypes()[1];
-        IAObject[] userMutableFields = new IAObject[] { new AMutableString(""), new AMutableString(""),
-                new AMutableInt32(0), new AMutableInt32(0), new AMutableString(""), new AMutableInt32(0) };
-        AUnorderedListType unorderedListType = new AUnorderedListType(BuiltinType.ASTRING, "referred-topics");
-        mutableFields = new IAObject[] { new AMutableString(""), new AMutableRecord(userRecordType, userMutableFields),
-                new AMutablePoint(0, 0), new AMutableDateTime(0), new AMutableUnorderedList(unorderedListType),
-                new AMutableString("") };
-        recordSerDe = new ARecordSerializerDeserializer(outputRecordType);
-        mutableRecord = new AMutableRecord(outputRecordType, mutableFields);
-
-    }
-
-    @Override
-    public void stop() {
-        // TODO Auto-generated method stub
-
+        return false;
     }
 
     public void setOutputStream(OutputStream os) {
         this.os = os;
     }
-}
\ No newline at end of file
+}
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/TweetGenerator2.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/TweetGenerator2.java
deleted file mode 100644
index b4db589..0000000
--- a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/TweetGenerator2.java
+++ /dev/null
@@ -1,166 +0,0 @@
-package edu.uci.ics.asterix.tools.external.data;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.util.Map;
-import java.util.logging.Logger;
-
-import edu.uci.ics.asterix.tools.external.data.DataGenerator2.InitializationInfo;
-import edu.uci.ics.asterix.tools.external.data.DataGenerator2.TweetMessage;
-import edu.uci.ics.asterix.tools.external.data.DataGenerator2.TweetMessageIterator;
-
-public class TweetGenerator2 {
-
-    private static final Logger LOGGER = Logger.getLogger(TweetGenerator.class.getName());
-
-    public static final String KEY_DURATION = "duration";
-    public static final String KEY_TPS = "tps";
-    public static final String KEY_MIN_TPS = "tps-min";
-    public static final String KEY_MAX_TPS = "tps-max";
-    public static final String KEY_TPUT_DURATION = "tput-duration";
-    public static final String KEY_GUID_SEED = "guid-seed";
-
-    public static final String OUTPUT_FORMAT = "output-format";
-    public static final String OUTPUT_FORMAT_ARECORD = "arecord";
-    public static final String OUTPUT_FORMAT_ADM_STRING = "adm-string";
-
-    private int duration;
-    private long tweetInterval;
-    private int numTweetsBeforeDelay;
-    private TweetMessageIterator tweetIterator = null;
-    private long exeptionInterval;
-    private int partition;
-    private int tweetCount = 0;
-    private int frameTweetCount = 0;
-    private int numFlushedTweets = 0;
-    private int exceptionPeriod = -1;
-    private boolean isOutputFormatRecord = false;
-    private byte[] EOL = "\n".getBytes();
-    private OutputStream os;
-    private DataGenerator2 dataGenerator = null;
-    private ByteBuffer outputBuffer = ByteBuffer.allocate(32 * 1024);
-    private int flushedTweetCount = 0;
-
-    public int getTweetCount() {
-        return tweetCount;
-    }
-
-    public TweetGenerator2(Map<String, String> configuration, int partition, String format) throws Exception {
-        String value = configuration.get(KEY_DURATION);
-        duration = value != null ? Integer.parseInt(value) : 60;
-        initializeTweetRate(configuration.get(KEY_TPS));
-        if (value != null) {
-            exceptionPeriod = Integer.parseInt(value);
-        }
-
-        isOutputFormatRecord = format.equalsIgnoreCase(OUTPUT_FORMAT_ARECORD);
-        InitializationInfo info = new InitializationInfo();
-        info.timeDurationInSecs = duration;
-        dataGenerator = new DataGenerator2(info);
-
-        String seedValue = configuration.get(KEY_GUID_SEED);
-        int seedInt = seedValue != null ? Integer.parseInt(seedValue) : 0;
-        tweetIterator = dataGenerator.new TweetMessageIterator(duration, partition, (byte) seedInt);
-    }
-
-    private void initializeTweetRate(String tps) {
-        numTweetsBeforeDelay = 0;
-        if (tps == null) {
-            tweetInterval = 0;
-        } else {
-            int val = Integer.parseInt(tps);
-            double interval = new Double(((double) 1000 / val));
-            if (interval > 1) {
-                tweetInterval = (long) interval;
-                numTweetsBeforeDelay = 1;
-            } else {
-                tweetInterval = 1;
-                Double numTweets = new Double(1 / interval);
-                if (numTweets.intValue() != numTweets) {
-                    tweetInterval = 5;
-                    numTweetsBeforeDelay = (new Double(10 * numTweets * 1)).intValue();
-                } else {
-                    numTweetsBeforeDelay = new Double((numTweets * 1)).intValue();
-                }
-            }
-        }
-
-    }
-
-    private void writeTweetString(TweetMessage next) throws IOException {
-        String tweet = next.toString() + "\n";
-        tweetCount++;
-        byte[] b = tweet.getBytes();
-        if (outputBuffer.position() + b.length > outputBuffer.limit()) {
-            flush();
-            numFlushedTweets += frameTweetCount;
-            frameTweetCount = 0;
-            flushedTweetCount += tweetCount - 1;
-            outputBuffer.put(tweet.getBytes());
-            frameTweetCount++;
-
-        } else {
-            outputBuffer.put(tweet.getBytes());
-            frameTweetCount++;
-        }
-    }
-
-    public int getNumFlushedTweets() {
-        return numFlushedTweets;
-    }
-
-    public int getFrameTweetCount() {
-        return frameTweetCount;
-    }
-
-    private void flush() throws IOException {
-        outputBuffer.flip();
-        os.write(outputBuffer.array(), 0, outputBuffer.limit());
-        outputBuffer.position(0);
-        outputBuffer.limit(32 * 1024);
-    }
-
-    private void writeTweetRecord(TweetMessage next) {
-        throw new UnsupportedOperationException("Invalid format");
-    }
-
-    public boolean setNextRecord() throws Exception {
-        boolean moreData = tweetIterator.hasNext();
-        if (!moreData) {
-            return false;
-        }
-        TweetMessage msg = tweetIterator.next();
-        if (isOutputFormatRecord) {
-            writeTweetRecord(msg);
-        } else {
-            writeTweetString(msg);
-        }
-        if (tweetInterval != 0) {
-            tweetCount++;
-            if (tweetCount == numTweetsBeforeDelay) {
-                Thread.sleep(tweetInterval);
-                tweetCount = 0;
-            }
-        }
-        return true;
-    }
-
-    public boolean setNextRecordBatch(int numTweetsInBatch) throws Exception {
-        int count = 0;
-        // if (tweetIterator.hasNext()) {
-        while (count < numTweetsInBatch) {
-            writeTweetString(tweetIterator.next());
-            count++;
-        }
-        // } else {
-        //   System.out.println("Flushing last batch, count so far:" + tweetCount);
-        // flush();
-        /// }
-        return true;
-    }
-
-    public void setOutputStream(OutputStream os) {
-        this.os = os;
-    }
-}
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/TwitterFirehoseFeedAdapter.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/TwitterFirehoseFeedAdapter.java
index be09685..07e018a 100644
--- a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/TwitterFirehoseFeedAdapter.java
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/TwitterFirehoseFeedAdapter.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package edu.uci.ics.asterix.tools.external.data;
 
 import java.io.IOException;
@@ -25,7 +39,6 @@
 
 /**
  * TPS can be configured between 1 and 20,000
- * 
  */
 public class TwitterFirehoseFeedAdapter extends StreamBasedAdapter implements IFeedAdapter {
 
@@ -39,6 +52,7 @@
 
     private static final String LOCALHOST = "127.0.0.1";
     private static final int PORT = 2909;
+    private static final int TPUT_DURATION_DEFAULT = 5; // 5 seconds
 
     private ExecutorService executorService = Executors.newCachedThreadPool();
 
@@ -52,7 +66,6 @@
     @Override
     public void start(int partition, IFrameWriter writer) throws Exception {
         twitterServer.start();
-        twitterServer.getListener().setPartition(partition);
         twitterClient.start();
         super.start(partition, writer);
     }
@@ -68,8 +81,8 @@
         private int port = -1;
         private ExecutorService executorService;
 
-        public TwitterServer(Map<String, String> configuration, ARecordType outputtype, ExecutorService executorService, int partition)
-                throws Exception {
+        public TwitterServer(Map<String, String> configuration, ARecordType outputtype,
+                ExecutorService executorService, int partition) throws Exception {
             int numAttempts = 0;
             while (port < 0) {
                 try {
@@ -90,10 +103,6 @@
             this.executorService = executorService;
         }
 
-        public Listener getListener() {
-            return listener;
-        }
-
         public void start() {
             executorService.execute(listener);
         }
@@ -106,7 +115,6 @@
         public int getPort() {
             return port;
         }
-
     }
 
     private static class TwitterClient {
@@ -132,13 +140,12 @@
 
         private final ServerSocket serverSocket;
         private Socket socket;
-        private TweetGenerator2 tweetGenerator;
+        private TweetGenerator tweetGenerator;
         private boolean continuePush = true;
         private int fixedTps = -1;
         private int minTps = -1;
         private int maxTps = -1;
         private int tputDuration;
-        private int partition;
         private Rate task;
         private Mode mode;
 
@@ -149,15 +156,10 @@
             CONTROLLED,
         }
 
-        public void setPartition(int partition) {
-            this.partition = partition;
-            task.setPartition(partition);
-        }
-
         public Listener(ServerSocket serverSocket, Map<String, String> configuration, ARecordType outputtype,
                 String datasetName, int partition) throws Exception {
             this.serverSocket = serverSocket;
-            this.tweetGenerator = new TweetGenerator2(configuration, partition, TweetGenerator.OUTPUT_FORMAT_ADM_STRING);
+            this.tweetGenerator = new TweetGenerator(configuration, partition, TweetGenerator.OUTPUT_FORMAT_ADM_STRING);
             String value = configuration.get(KEY_MODE);
             String confValue = null;
             if (value != null) {
@@ -166,19 +168,19 @@
                     case AGGRESSIVE:
                         break;
                     case CONTROLLED:
-                        confValue = configuration.get(TweetGenerator2.KEY_TPS);
+                        confValue = configuration.get(TweetGenerator.KEY_TPS);
                         if (confValue != null) {
                             minTps = Integer.parseInt(confValue);
                             maxTps = minTps;
                             fixedTps = minTps;
                         } else {
-                            confValue = configuration.get(TweetGenerator2.KEY_MIN_TPS);
+                            confValue = configuration.get(TweetGenerator.KEY_MIN_TPS);
                             if (confValue != null) {
                                 minTps = Integer.parseInt(confValue);
                             }
-                            confValue = configuration.get(TweetGenerator2.KEY_MAX_TPS);
+                            confValue = configuration.get(TweetGenerator.KEY_MAX_TPS);
                             if (confValue != null) {
-                                maxTps = Integer.parseInt(configuration.get(TweetGenerator2.KEY_MAX_TPS));
+                                maxTps = Integer.parseInt(configuration.get(TweetGenerator.KEY_MAX_TPS));
                             }
 
                             if (minTps < 0 || maxTps < 0 || minTps > maxTps) {
@@ -191,9 +193,9 @@
                 mode = Mode.AGGRESSIVE;
             }
 
-            tputDuration = Integer.parseInt(configuration.get(TweetGenerator2.KEY_TPUT_DURATION));
+            value = configuration.get(TweetGenerator.KEY_TPUT_DURATION);
+            tputDuration = value != null ? Integer.parseInt(value) : TPUT_DURATION_DEFAULT;
             task = new Rate(tweetGenerator, tputDuration, datasetName, partition);
-
         }
 
         @Override
@@ -209,20 +211,25 @@
                     long startBatch;
                     long endBatch;
                     Random random = new Random();
-                    int tps = 0;
+                    int batchSize = 0;
                     while (moreData && continuePush) {
-                        if(maxTps > 0){
-                             tps = minTps + random.nextInt((maxTps+1) - minTps);   
-                        } else {
-                            tps = fixedTps;
-                        }
-                        startBatch = System.currentTimeMillis();
-                        moreData = tweetGenerator.setNextRecordBatch(tps);
-                        endBatch = System.currentTimeMillis();
-                        if (mode.equals(Mode.CONTROLLED)) {
-                            if (endBatch - startBatch < 1000) {
-                                Thread.sleep(1000 - (endBatch - startBatch));
-                            }
+                        switch (mode) {
+                            case CONTROLLED:
+                                if (maxTps > 0) {
+                                    batchSize = minTps + random.nextInt((maxTps + 1) - minTps);
+                                } else {
+                                    batchSize = fixedTps;
+                                }
+                                startBatch = System.currentTimeMillis();
+                                moreData = tweetGenerator.setNextRecordBatch(batchSize);
+                                endBatch = System.currentTimeMillis();
+                                if (endBatch - startBatch < 1000) {
+                                    Thread.sleep(1000 - (endBatch - startBatch));
+                                }
+                                break;
+                            case AGGRESSIVE:
+                                batchSize = Integer.MAX_VALUE;
+                                moreData = tweetGenerator.setNextRecordBatch(batchSize);
                         }
                     }
                     timer.cancel();
@@ -254,16 +261,17 @@
 
         private static class Rate extends TimerTask {
 
-            private TweetGenerator2 gen;
-            int prevMeasuredTweets = 0;
-            private int tputDuration;
-            private int partition;
-            private String dataset;
+            private final TweetGenerator gen;
+            private final int tputDuration;
+            private final int partition;
+            private final String dataset;
+            private int prevMeasuredTweets = 0;
 
-            public Rate(TweetGenerator2 gen, int tputDuration, String dataset, int partition) {
+            public Rate(TweetGenerator gen, int tputDuration, String dataset, int partition) {
                 this.gen = gen;
                 this.tputDuration = tputDuration;
                 this.dataset = dataset;
+                this.partition = partition;
                 if (LOGGER.isLoggable(Level.WARNING)) {
                     LOGGER.warning(new Date() + " " + "Dataset" + " " + "partition" + " " + "Total flushed tweets"
                             + "\t" + "intantaneous throughput");
@@ -272,22 +280,15 @@
 
             @Override
             public void run() {
-
                 int currentMeasureTweets = gen.getNumFlushedTweets();
-
                 if (LOGGER.isLoggable(Level.FINE)) {
                     LOGGER.fine(dataset + " " + partition + " " + gen.getNumFlushedTweets() + "\t"
                             + ((currentMeasureTweets - prevMeasuredTweets) / tputDuration) + " ID "
                             + Thread.currentThread().getId());
                 }
-
                 prevMeasuredTweets = currentMeasureTweets;
-
             }
 
-            public void setPartition(int partition) {
-                this.partition = partition;
-            }
         }
     }
 
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/TwitterFirehoseFeedAdapterFactory.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/TwitterFirehoseFeedAdapterFactory.java
index b18a5f2..2305c32 100644
--- a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/TwitterFirehoseFeedAdapterFactory.java
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/external/data/TwitterFirehoseFeedAdapterFactory.java
@@ -17,7 +17,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
 import edu.uci.ics.asterix.external.adapter.factory.StreamBasedAdapterFactory;
@@ -29,7 +28,6 @@
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.asterix.om.util.AsterixClusterProperties;
 import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
 import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
 import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;