remove minidfs cluster (so writing directly to lcoal disk now)
diff --git a/genomix/genomix-data/src/main/java/edu/uci/ics/genomix/minicluster/GenomixMiniCluster.java b/genomix/genomix-data/src/main/java/edu/uci/ics/genomix/minicluster/GenomixMiniCluster.java
index 09f7db9..abaac30 100644
--- a/genomix/genomix-data/src/main/java/edu/uci/ics/genomix/minicluster/GenomixMiniCluster.java
+++ b/genomix/genomix-data/src/main/java/edu/uci/ics/genomix/minicluster/GenomixMiniCluster.java
@@ -1,32 +1,91 @@
package edu.uci.ics.genomix.minicluster;
-import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.FileSystems;
import java.nio.file.Files;
-import java.nio.file.attribute.FileAttribute;
import java.nio.file.Path;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.wicket.util.file.File;
import edu.uci.ics.pregelix.core.jobgen.clusterconfig.ClusterConfig;
-import edu.uci.ics.pregelix.core.util.PregelixHyracksIntegrationUtil;
+import edu.uci.ics.pregelix.runtime.bootstrap.NCApplicationEntryPoint;
import edu.uci.ics.genomix.config.GenomixJobConf;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.common.controllers.CCConfig;
+import edu.uci.ics.hyracks.control.common.controllers.NCConfig;
+import edu.uci.ics.hyracks.control.nc.NodeControllerService;
-@SuppressWarnings("deprecation")
public class GenomixMiniCluster {
- private MiniDFSCluster dfsCluster;
- private Path clusterProperties;
- private Path clusterStores;
- private Path clusterWorkDir;
- private Path clusterStoresDir;
+
+ public static final String NC1_ID = "nc1";
+ public static final int TEST_HYRACKS_CC_PORT = 1099;
+ public static final int TEST_HYRACKS_CC_CLIENT_PORT = 2099;
+ public static final String CC_HOST = "localhost";
- private void makeLocalClusterConfig() throws IOException {
+ private static ClusterControllerService cc;
+ private static NodeControllerService nc1;
+ private static MiniDFSCluster dfsCluster;
+ private static Path clusterProperties;
+ private static Path clusterStores;
+ private static Path clusterWorkDir;
+ private static Path clusterStoresDir;
+
+ public static void init(GenomixJobConf conf) throws Exception {
+ makeLocalClusterConfig();
+ ClusterConfig.setClusterPropertiesPath(clusterProperties.toAbsolutePath().toString());
+ ClusterConfig.setStorePath(clusterStores.toAbsolutePath().toString());
+// dfsCluster = new MiniDFSCluster(conf, Integer.parseInt(conf.get(GenomixJobConf.CPARTITION_PER_MACHINE)), true, null);
+
+ // cluster controller
+ CCConfig ccConfig = new CCConfig();
+ ccConfig.clientNetIpAddress = CC_HOST;
+ ccConfig.clusterNetIpAddress = CC_HOST;
+ ccConfig.clusterNetPort = TEST_HYRACKS_CC_PORT;
+ ccConfig.clientNetPort = TEST_HYRACKS_CC_CLIENT_PORT;
+ ccConfig.defaultMaxJobAttempts = 0;
+ ccConfig.jobHistorySize = 1;
+ ccConfig.profileDumpPeriod = -1;
+ cc = new ClusterControllerService(ccConfig);
+ cc.start();
+
+ // one node controller
+ NCConfig ncConfig1 = new NCConfig();
+ ncConfig1.ccHost = "localhost";
+ ncConfig1.clusterNetIPAddress = "localhost";
+ ncConfig1.ccPort = TEST_HYRACKS_CC_PORT;
+ ncConfig1.dataIPAddress = "127.0.0.1";
+ ncConfig1.datasetIPAddress = "127.0.0.1";
+ ncConfig1.nodeId = NC1_ID;
+ ncConfig1.ioDevices = clusterWorkDir.toString() + File.separator + "tmp" + File.separator + "t3";
+ ncConfig1.appNCMainClass = NCApplicationEntryPoint.class.getName();
+ nc1 = new NodeControllerService(ncConfig1);
+ nc1.start();
+ }
+
+ public static void deinit() throws Exception {
+ nc1.stop();
+ cc.stop();
+// dfsCluster.shutdown();
+ FileUtils.deleteQuietly(new File("build"));
+ FileUtils.deleteQuietly(clusterProperties.toFile());
+ FileUtils.deleteQuietly(clusterStores.toFile());
+ FileUtils.deleteQuietly(clusterWorkDir.toFile());
+ FileUtils.deleteQuietly(clusterStoresDir.toFile());
+ }
+
+ /**
+ * create the necessary .properties and directories for a minicluster instance
+ */
+ private static void makeLocalClusterConfig() throws IOException {
clusterProperties = Files.createTempFile(FileSystems.getDefault().getPath("."), "tmp.cluster", ".properties");
clusterWorkDir = Files.createTempDirectory(FileSystems.getDefault().getPath("."), "tmp.clusterWorkDir");
PrintWriter writer = new PrintWriter(clusterProperties.toString(), "US-ASCII");
- writer.println(String.format("WORKPATH=\"%s\"", clusterWorkDir.toAbsolutePath().toString()));
+ writer.println("CC_CLIENTPORT=" + TEST_HYRACKS_CC_CLIENT_PORT);
+ writer.println("CC_CLUSTERPORT=" + TEST_HYRACKS_CC_PORT);
+ writer.println(String.format("WORKPATH=%s", clusterWorkDir.toAbsolutePath().toString()));
writer.println("CCTMP_DIR=${WORKPATH}/tmp/t1");
writer.println("NCTMP_DIR=${WORKPATH}/tmp/t2");
writer.println("CCLOGS_DIR=$CCTMP_DIR/logs");
@@ -42,21 +101,7 @@
clusterStoresDir = Files.createTempDirectory(FileSystems.getDefault().getPath("."), "tmp.clusterStores");
clusterStores = Files.createTempFile(FileSystems.getDefault().getPath("."), "tmp.stores", ".properties");
writer = new PrintWriter(clusterStores.toString(), "US-ASCII");
- writer.println(String.format("store=\"%s\"", clusterStoresDir.toAbsolutePath().toString()));
+ writer.println(String.format("store=%s", clusterStoresDir.toString()));
writer.close();
}
-
- public void setUp(GenomixJobConf conf, String clusterStore, String clusterProperties) throws Exception {
- makeLocalClusterConfig();
- ClusterConfig.setClusterPropertiesPath(clusterPropertiesDir.toAbsolutePath().toString() + File.separator + "cluster.properties");
- ClusterConfig.setStorePath(clusterPropertiesDir.toAbsolutePath().toString() + File.separator + "stores.properties");
- PregelixHyracksIntegrationUtil.init();
- dfsCluster = new MiniDFSCluster(conf, Integer.parseInt(conf.get(GenomixJobConf.CPARTITION_PER_MACHINE)), true,
- null);
- }
-
- public void tearDown() throws Exception {
- PregelixHyracksIntegrationUtil.deinit();
- dfsCluster.shutdown();
- }
}
diff --git a/genomix/genomix-driver/data/AddBridge/SimpleTest/text.txt b/genomix/genomix-driver/data/AddBridge/SimpleTest/text.txt
deleted file mode 100755
index 01c49e5..0000000
--- a/genomix/genomix-driver/data/AddBridge/SimpleTest/text.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-1 AATAGAAG
-2 AATAGCTT
-3 AATAGAAG
-4 AATAGCTT
-5 AATAGAAG
-6 AGAAGAAG
diff --git a/genomix/genomix-driver/src/main/java/edu/uci/ics/genomix/driver/GenomixDriver.java b/genomix/genomix-driver/src/main/java/edu/uci/ics/genomix/driver/GenomixDriver.java
index 0ca88ec..cd31ffe 100644
--- a/genomix/genomix-driver/src/main/java/edu/uci/ics/genomix/driver/GenomixDriver.java
+++ b/genomix/genomix-driver/src/main/java/edu/uci/ics/genomix/driver/GenomixDriver.java
@@ -186,12 +186,11 @@
public void runGenomix(GenomixJobConf conf) throws NumberFormatException, HyracksException, Exception {
KmerBytesWritable.setGlobalKmerLength(Integer.parseInt(conf.get(GenomixJobConf.KMER_LENGTH)));
- GenomixMiniCluster testCluster = new GenomixMiniCluster();
jobs = new ArrayList<PregelixJob>();
stepNum = 0;
boolean runLocal = Boolean.parseBoolean(conf.get(GenomixJobConf.RUN_LOCAL));
if (runLocal)
- testCluster.setUp(conf);
+ GenomixMiniCluster.init(conf);
String localInput = conf.get(GenomixJobConf.LOCAL_INPUT_DIR);
if (localInput != null) {
@@ -272,7 +271,7 @@
FileSystem.get(conf).rename(new Path(curOutput), new Path(GenomixJobConf.FINAL_OUTPUT_DIR));
if (runLocal)
- testCluster.tearDown();
+ GenomixMiniCluster.deinit();
}
public static void main(String[] args) throws CmdLineException, NumberFormatException, HyracksException, Exception {
@@ -282,7 +281,7 @@
// "-inputDir", "/home/wbiesing/code/hyracks/genomix/genomix-driver/graphbuild.binmerge",
// "-localInput", "../genomix-pregelix/data/TestSet/PathMerge/CyclePath/bin/part-00000",
"-pipelineOrder", "BUILD" };
- GenomixJobConf conf = GenomixJobConf.fromArguments(args);
+ GenomixJobConf conf = GenomixJobConf.fromArguments(myArgs);
GenomixDriver driver = new GenomixDriver();
driver.runGenomix(conf);
}