make genomix compile with merged hyracks
diff --git a/genomix/genomix-data/.classpath b/genomix/genomix-data/.classpath
index e43402f..6e26406 100644
--- a/genomix/genomix-data/.classpath
+++ b/genomix/genomix-data/.classpath
@@ -1,36 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java">
- <attributes>
- <attribute name="optional" value="true"/>
- <attribute name="maven.pomderived" value="true"/>
- </attributes>
- </classpathentry>
- <classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
- <attributes>
- <attribute name="maven.pomderived" value="true"/>
- </attributes>
- </classpathentry>
- <classpathentry kind="src" output="target/test-classes" path="src/test/java">
- <attributes>
- <attribute name="optional" value="true"/>
- <attribute name="maven.pomderived" value="true"/>
- </attributes>
- </classpathentry>
- <classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
- <attributes>
- <attribute name="maven.pomderived" value="true"/>
- </attributes>
- </classpathentry>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
- <attributes>
- <attribute name="maven.pomderived" value="true"/>
- </attributes>
- </classpathentry>
- <classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
- <attributes>
- <attribute name="maven.pomderived" value="true"/>
- </attributes>
- </classpathentry>
+ <classpathentry kind="src" output="target/classes" path="src/main/java"/>
+ <classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources"/>
+ <classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
+ <classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>
+ <classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER"/>
<classpathentry kind="output" path="target/classes"/>
</classpath>
diff --git a/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/driver/Driver.java b/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/driver/Driver.java
index 27066a2..99d6c89 100644
--- a/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/driver/Driver.java
+++ b/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/driver/Driver.java
@@ -108,7 +108,6 @@
private void execute(JobSpecification job) throws Exception {
job.setUseConnectorPolicyForScheduling(false);
JobId jobId = hcc.startJob(
- applicationName,
job,
profiling ? EnumSet.of(JobFlag.PROFILE_RUNTIME) : EnumSet
.noneOf(JobFlag.class));
diff --git a/genomix/genomix-hyracks/src/test/java/edu/uci/ics/genomix/example/jobrun/JobRunTest.java b/genomix/genomix-hyracks/src/test/java/edu/uci/ics/genomix/example/jobrun/JobRunTest.java
index 5a64f78..675de64 100644
--- a/genomix/genomix-hyracks/src/test/java/edu/uci/ics/genomix/example/jobrun/JobRunTest.java
+++ b/genomix/genomix-hyracks/src/test/java/edu/uci/ics/genomix/example/jobrun/JobRunTest.java
@@ -62,7 +62,6 @@
public void setUp() throws Exception {
cleanupStores();
HyracksUtils.init();
- HyracksUtils.createApp(HYRACKS_APP_NAME);
FileUtils.forceMkdir(new File(ACTUAL_RESULT_DIR));
FileUtils.cleanDirectory(new File(ACTUAL_RESULT_DIR));
startHDFS();
@@ -97,7 +96,7 @@
Path dest = new Path(HDFS_INPUT_PATH);
Path result = new Path(HDFS_OUTPUT_PATH);
dfs.mkdirs(dest);
- //dfs.mkdirs(result);
+ // dfs.mkdirs(result);
dfs.copyFromLocalFile(src, dest);
DataOutputStream confOutput = new DataOutputStream(
@@ -119,7 +118,7 @@
}
@Test
- public void TestAll() throws Exception{
+ public void TestAll() throws Exception {
cleanUpReEntry();
TestExternalGroupby();
cleanUpReEntry();
@@ -134,7 +133,7 @@
cleanUpReEntry();
TestHybridReversedGroupby();
}
-
+
public void TestExternalGroupby() throws Exception {
conf.set(GenomixJob.GROUPBY_TYPE, "external");
System.err.println("Testing ExternalGroupBy");
@@ -155,22 +154,24 @@
driver.runJob(new GenomixJob(conf), Plan.BUILD_DEBRUJIN_GRAPH, true);
Assert.assertEquals(true, checkResults(EXPECTED_PATH));
}
-
- public void TestExternalReversedGroupby() throws Exception{
+
+ public void TestExternalReversedGroupby() throws Exception {
conf.set(GenomixJob.GROUPBY_TYPE, "external");
conf.setBoolean(GenomixJob.REVERSED_KMER, true);
System.err.println("Testing ExternalGroupBy + Reversed");
driver.runJob(new GenomixJob(conf), Plan.BUILD_DEBRUJIN_GRAPH, true);
Assert.assertEquals(true, checkResults(EXPECTED_REVERSE_PATH));
}
- public void TestPreClusterReversedGroupby() throws Exception{
+
+ public void TestPreClusterReversedGroupby() throws Exception {
conf.set(GenomixJob.GROUPBY_TYPE, "precluster");
conf.setBoolean(GenomixJob.REVERSED_KMER, true);
System.err.println("Testing PreclusterGroupBy + Reversed");
driver.runJob(new GenomixJob(conf), Plan.BUILD_DEBRUJIN_GRAPH, true);
Assert.assertEquals(true, checkResults(EXPECTED_REVERSE_PATH));
}
- public void TestHybridReversedGroupby() throws Exception{
+
+ public void TestHybridReversedGroupby() throws Exception {
conf.set(GenomixJob.GROUPBY_TYPE, "hybrid");
conf.setBoolean(GenomixJob.REVERSED_KMER, true);
System.err.println("Testing HybridGroupBy + Reversed");
@@ -188,21 +189,21 @@
DUMPED_RESULT), false, conf, null);
dumped = new File(DUMPED_RESULT);
} else {
-
- FileSystem.getLocal(new Configuration()).mkdirs(new Path(ACTUAL_RESULT_DIR
- + HDFS_OUTPUT_PATH));
+
+ FileSystem.getLocal(new Configuration()).mkdirs(
+ new Path(ACTUAL_RESULT_DIR + HDFS_OUTPUT_PATH));
File filePathTo = new File(CONVERT_RESULT);
BufferedWriter bw = new BufferedWriter(new FileWriter(filePathTo));
for (int i = 0; i < numPartitionPerMachine * numberOfNC; i++) {
String partname = "/part-" + i;
- FileUtil.copy(FileSystem.get(conf), new Path(HDFS_OUTPUT_PATH
- + partname), FileSystem.getLocal(new Configuration()),
- new Path(ACTUAL_RESULT_DIR + HDFS_OUTPUT_PATH + partname), false, conf);
-
- Path path = new Path(HDFS_OUTPUT_PATH
- + partname);
+ FileUtil.copy(FileSystem.get(conf), new Path(HDFS_OUTPUT_PATH
+ + partname), FileSystem.getLocal(new Configuration()),
+ new Path(ACTUAL_RESULT_DIR + HDFS_OUTPUT_PATH
+ + partname), false, conf);
+
+ Path path = new Path(HDFS_OUTPUT_PATH + partname);
FileSystem dfs = FileSystem.get(conf);
- if (dfs.getFileStatus(path).getLen() == 0){
+ if (dfs.getFileStatus(path).getLen() == 0) {
continue;
}
SequenceFile.Reader reader = new SequenceFile.Reader(dfs, path,
@@ -214,15 +215,14 @@
int k = conf.getInt(GenomixJob.KMER_LENGTH, 25);
while (reader.next(key, value)) {
- if (key == null || value == null){
+ if (key == null || value == null) {
break;
}
bw.write(Kmer.recoverKmerFrom(k, key.getBytes(), 0,
key.getLength())
+ "\t" + value.toString());
- System.out.println(Kmer.recoverKmerFrom(k, key.getBytes(), 0,
- key.getLength())
- + "\t" + value.toString());
+ System.out.println(Kmer.recoverKmerFrom(k, key.getBytes(),
+ 0, key.getLength()) + "\t" + value.toString());
bw.newLine();
}
reader.close();
@@ -232,13 +232,12 @@
dumped = new File(CONVERT_RESULT);
}
- TestUtils.compareWithSortedResult(new File(expectedPath), dumped);
+ TestUtils.compareWithResult(new File(expectedPath), dumped);
return true;
}
@After
public void tearDown() throws Exception {
- HyracksUtils.destroyApp(HYRACKS_APP_NAME);
HyracksUtils.deinit();
cleanupHDFS();
}
diff --git a/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/JobRun/RunJobTestSuite.java b/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/JobRun/RunJobTestSuite.java
index 0238ca0..ba0f4c3 100644
--- a/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/JobRun/RunJobTestSuite.java
+++ b/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/JobRun/RunJobTestSuite.java
@@ -25,8 +25,8 @@
import edu.uci.ics.pregelix.core.jobgen.clusterconfig.ClusterConfig;
import edu.uci.ics.pregelix.core.util.PregelixHyracksIntegrationUtil;
-public class RunJobTestSuite extends TestSuite{
-
+public class RunJobTestSuite extends TestSuite {
+
private static final Logger LOGGER = Logger.getLogger(RunJobTestSuite.class
.getName());
@@ -40,9 +40,9 @@
private static final String PATH_TO_ONLY = "src/test/resources/only.txt";
private static final String FILE_EXTENSION_OF_RESULTS = "result";
- private static final String DATA_PATH = "data/input/part-1-out-500000";//sequenceShortFileMergeTest
+ private static final String DATA_PATH = "data/input/part-1-out-500000";// sequenceShortFileMergeTest
private static final String HDFS_PATH = "/webmap/";
-
+
private static final String HYRACKS_APP_NAME = "pregelix";
private static final String HADOOP_CONF_PATH = ACTUAL_RESULT_DIR
+ File.separator + "conf.xml";
@@ -50,13 +50,12 @@
private JobConf conf = new JobConf();
private int numberOfNC = 2;
-
+
public void setUp() throws Exception {
ClusterConfig.setStorePath(PATH_TO_CLUSTER_STORE);
ClusterConfig.setClusterPropertiesPath(PATH_TO_CLUSTER_PROPERTIES);
cleanupStores();
PregelixHyracksIntegrationUtil.init("src/test/resources/topology.xml");
- PregelixHyracksIntegrationUtil.createApp(HYRACKS_APP_NAME);
LOGGER.info("Hyracks mini-cluster started");
FileUtils.forceMkdir(new File(ACTUAL_RESULT_DIR));
FileUtils.cleanDirectory(new File(ACTUAL_RESULT_DIR));
@@ -69,7 +68,7 @@
FileUtils.cleanDirectory(new File("teststore"));
FileUtils.cleanDirectory(new File("build"));
}
-
+
private void startHDFS() throws IOException {
conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml"));
conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
@@ -90,7 +89,7 @@
confOutput.flush();
confOutput.close();
}
-
+
/**
* cleanup hdfs cluster
*/
@@ -99,12 +98,11 @@
}
public void tearDown() throws Exception {
- PregelixHyracksIntegrationUtil.destroyApp(HYRACKS_APP_NAME);
PregelixHyracksIntegrationUtil.deinit();
LOGGER.info("Hyracks mini-cluster shut down");
cleanupHDFS();
}
-
+
public static Test suite() throws Exception {
List<String> ignores = getFileList(PATH_TO_IGNORE);
List<String> onlys = getFileList(PATH_TO_ONLY);
@@ -138,12 +136,12 @@
}
return testSuite;
}
-
+
/**
* Runs the tests and collects their result in a TestResult.
*/
@Override
- public void run(TestResult result) {
+ public void run(TestResult result) {
try {
int testCount = countTestCases();
for (int i = 0; i < testCount; i++) {
@@ -158,8 +156,7 @@
} catch (Exception e) {
throw new IllegalStateException(e);
}
-
-
+
}
protected static List<String> getFileList(String ignorePath)
diff --git a/pom.xml b/pom.xml
index 99cdc4c..eec955c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -68,5 +68,6 @@
<module>algebricks</module>
<module>pregelix</module>
<module>hivesterix</module>
+ <module>genomix<module>
</modules>
</project>