add more testcase
diff --git a/genomix/genomix-pregelix/pom.xml b/genomix/genomix-pregelix/pom.xml
index 854335f..bee0907 100644
--- a/genomix/genomix-pregelix/pom.xml
+++ b/genomix/genomix-pregelix/pom.xml
@@ -135,6 +135,95 @@
<type>jar</type>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-dataflow-std</artifactId>
+ <version>0.2.6-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-api</artifactId>
+ <version>0.2.6-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-dataflow-common</artifactId>
+ <version>0.2.6-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-data-std</artifactId>
+ <version>0.2.6-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-control-cc</artifactId>
+ <version>0.2.6-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-control-nc</artifactId>
+ <version>0.2.6-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.kenai.nbpwr</groupId>
+ <artifactId>org-apache-commons-io</artifactId>
+ <version>1.3.1-201002241208</version>
+ <type>nbm</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks.examples</groupId>
+ <artifactId>hyracks-integration-tests</artifactId>
+ <version>0.2.6-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-ipc</artifactId>
+ <version>0.2.6-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ <version>0.20.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-test</artifactId>
+ <version>0.20.2</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-hdfs-core</artifactId>
+ <version>0.2.6-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-hdfs-core</artifactId>
+ <version>0.2.6-SNAPSHOT</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>genomix-hyracks</artifactId>
+ <version>0.2.6-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
<scm>
diff --git a/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/operator/NaiveAlgorithmForPathMergeVertex.java b/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/operator/NaiveAlgorithmForPathMergeVertex.java
index 4a55889..d75ea48 100644
--- a/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/operator/NaiveAlgorithmForPathMergeVertex.java
+++ b/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/operator/NaiveAlgorithmForPathMergeVertex.java
@@ -143,11 +143,13 @@
sendMsg(destVertexId,msg);
}
}else{// is Rear
- getVertexValue().set(GraphVertexOperation.updateRightNeighberByVertexId(getVertexValue().getAdjMap(), msg.getSourceVertexId(), kmerSize),
- State.FINAL_VERTEX, msg.getChainVertexId());
- setVertexValue(getVertexValue());
- //String source = msg.getChainVertexId().toString();
- //System.out.print("");
+ if(msg.getLengthOfChain() > kmerSize){
+ byte tmp = GraphVertexOperation.updateRightNeighberByVertexId(getVertexValue().getAdjMap(), msg.getSourceVertexId(), kmerSize);
+ getVertexValue().set(tmp, State.FINAL_VERTEX, msg.getChainVertexId());
+ setVertexValue(getVertexValue());
+ //String source = msg.getChainVertexId().toString();
+ //System.out.print("");
+ }
}
}
@@ -169,8 +171,12 @@
//head node sends message to path node
else if(getSuperstep()%2 == 1 && getSuperstep() <= maxIteration){
while (msgIterator.hasNext()){
- msg = msgIterator.next();
- sendMsgToPathVertex();
+ if(getSuperstep() == 3 && GraphVertexOperation.isRearVertex(getVertexValue().getAdjMap()))
+ voteToHalt();
+ else{
+ msg = msgIterator.next();
+ sendMsgToPathVertex();
+ }
}
}
//path node sends message back to head node
diff --git a/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/testcase/GenerateTestInput.java b/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/testcase/GenerateTestInput.java
index 6d81b74..fc26f66 100644
--- a/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/testcase/GenerateTestInput.java
+++ b/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/testcase/GenerateTestInput.java
@@ -56,9 +56,9 @@
OutputStreamWriter writer;
try {
writer = new OutputStreamWriter(new FileOutputStream("graph/55/SinglePath_55"));
- writer.write(simplePath(55,60,1));
+ writer.write(simplePath(55,500,1));
writer.close();
- writer = new OutputStreamWriter(new FileOutputStream("graph/55/SimplePath_55"));
+ /*writer = new OutputStreamWriter(new FileOutputStream("graph/55/SimplePath_55"));
writer.write(simplePath(55,60,3));
writer.close();
writer = new OutputStreamWriter(new FileOutputStream("graph/55/TreePath"));
@@ -69,7 +69,7 @@
writer.close();
writer = new OutputStreamWriter(new FileOutputStream("graph/55/BridgePath"));
writer.write(bridgePath(55,2));
- writer.close();
+ writer.close();*/
} catch (IOException e) {
e.printStackTrace();
}
diff --git a/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/JobRun/RunJobTestSuite.java b/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/JobRun/RunJobTestSuite.java
index fd0749a..45d8185 100644
--- a/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/JobRun/RunJobTestSuite.java
+++ b/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/JobRun/RunJobTestSuite.java
@@ -40,7 +40,7 @@
private static final String PATH_TO_ONLY = "src/test/resources/only.txt";
private static final String FILE_EXTENSION_OF_RESULTS = "result";
- private static final String DATA_PATH = "data/sequencefile/LongPath";
+ private static final String DATA_PATH = "data/sequencefile/TreePath";
private static final String HDFS_PATH = "/webmap/";
private static final String HYRACKS_APP_NAME = "pregelix";
diff --git a/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/ResultGen/ReportGenerator.java b/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/ResultGen/ReportGenerator.java
index 4bb37f7..3281a3b 100644
--- a/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/ResultGen/ReportGenerator.java
+++ b/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/ResultGen/ReportGenerator.java
@@ -44,5 +44,7 @@
generateReportFromLoginfo("naive");
generateReportFromLoginfo("log");
generateReportFromLoginfo("log_yourkit");
+ generateReportFromLoginfo("naive_36");
+ generateReportFromLoginfo("log_13");
}
}
diff --git a/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/GraphBuildTest.java b/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/GraphBuildTest.java
new file mode 100644
index 0000000..7ecf31f
--- /dev/null
+++ b/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/GraphBuildTest.java
@@ -0,0 +1,188 @@
+package edu.uci.ics.genomix.pregelix.pathmerge;
+
+import java.io.BufferedWriter;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import edu.uci.ics.genomix.driver.Driver;
+import edu.uci.ics.genomix.driver.Driver.Plan;
+import edu.uci.ics.genomix.example.jobrun.TestUtils;
+import edu.uci.ics.genomix.job.GenomixJob;
+import edu.uci.ics.genomix.type.KmerBytesWritable;
+import edu.uci.ics.genomix.type.KmerCountValue;
+
+public class GraphBuildTest {
+ private static final String ACTUAL_RESULT_DIR = "graphbuildresult";
+ private static final String PATH_TO_HADOOP_CONF = "src/test/resources/hadoop/conf";
+
+ private static final String DATA_PATH = "data/sequencefile/BridgePath";
+ private static final String HDFS_INPUT_PATH = "/BridgePath";
+ private static final String HDFS_OUTPUT_PATH = "/BridgePath_result";
+
+ private static final String DUMPED_RESULT = ACTUAL_RESULT_DIR + HDFS_OUTPUT_PATH + "/result.txt";
+ private static final String CONVERT_RESULT = DUMPED_RESULT + ".txt";
+ private static final String EXPECTED_PATH = "src/test/resources/expected/result2";
+ private static final String EXPECTED_REVERSE_PATH = "src/test/resources/expected/result_reverse";
+
+ private static final String HADOOP_CONF_PATH = ACTUAL_RESULT_DIR + File.separator + "conf.xml";
+ private MiniDFSCluster dfsCluster;
+
+ private JobConf conf = new JobConf();
+ private int numberOfNC = 2;
+ private int numPartitionPerMachine = 1;
+
+ private Driver driver;
+
+ @Before
+ public void setUp() throws Exception {
+ cleanupStores();
+ edu.uci.ics.hyracks.hdfs.utils.HyracksUtils.init();
+ FileUtils.forceMkdir(new File(ACTUAL_RESULT_DIR));
+ FileUtils.cleanDirectory(new File(ACTUAL_RESULT_DIR));
+ startHDFS();
+
+ FileInputFormat.setInputPaths(conf, HDFS_INPUT_PATH);
+ FileOutputFormat.setOutputPath(conf, new Path(HDFS_OUTPUT_PATH));
+
+ conf.setInt(GenomixJob.KMER_LENGTH, 5);
+ driver = new Driver(edu.uci.ics.hyracks.hdfs.utils.HyracksUtils.CC_HOST,
+ edu.uci.ics.hyracks.hdfs.utils.HyracksUtils.TEST_HYRACKS_CC_CLIENT_PORT, numPartitionPerMachine);
+ }
+
+ private void cleanupStores() throws IOException {
+ FileUtils.forceMkdir(new File("teststore"));
+ FileUtils.forceMkdir(new File("build"));
+ FileUtils.cleanDirectory(new File("teststore"));
+ FileUtils.cleanDirectory(new File("build"));
+ }
+
+ private void startHDFS() throws IOException {
+ conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml"));
+ conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
+ conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));
+
+ FileSystem lfs = FileSystem.getLocal(new Configuration());
+ lfs.delete(new Path("build"), true);
+ System.setProperty("hadoop.log.dir", "logs");
+ dfsCluster = new MiniDFSCluster(conf, numberOfNC, true, null);
+ FileSystem dfs = FileSystem.get(conf);
+ Path src = new Path(DATA_PATH);
+ Path dest = new Path(HDFS_INPUT_PATH);
+ dfs.mkdirs(dest);
+ dfs.copyFromLocalFile(src, dest);
+
+ DataOutputStream confOutput = new DataOutputStream(new FileOutputStream(new File(HADOOP_CONF_PATH)));
+ conf.writeXml(confOutput);
+ confOutput.flush();
+ confOutput.close();
+ }
+
+ private void cleanUpReEntry() throws IOException {
+ FileSystem lfs = FileSystem.getLocal(new Configuration());
+ if (lfs.exists(new Path(DUMPED_RESULT))) {
+ lfs.delete(new Path(DUMPED_RESULT), true);
+ }
+ FileSystem dfs = FileSystem.get(conf);
+ if (dfs.exists(new Path(HDFS_OUTPUT_PATH))) {
+ dfs.delete(new Path(HDFS_OUTPUT_PATH), true);
+ }
+ }
+
+ @Test
+ public void TestAll() throws Exception {
+ cleanUpReEntry();
+ TestHybridGroupby();
+ cleanUpReEntry();
+ TestPreClusterGroupby();
+ }
+
+ public void TestPreClusterGroupby() throws Exception {
+ conf.set(GenomixJob.GROUPBY_TYPE, "precluster");
+ System.err.println("Testing PreClusterGroupBy");
+ driver.runJob(new GenomixJob(conf), Plan.BUILD_DEBRUJIN_GRAPH, true);
+ Assert.assertEquals(true, checkResults(EXPECTED_PATH));
+ }
+
+ public void TestHybridGroupby() throws Exception {
+ conf.set(GenomixJob.GROUPBY_TYPE, "hybrid");
+ System.err.println("Testing HybridGroupBy");
+ driver.runJob(new GenomixJob(conf), Plan.BUILD_DEBRUJIN_GRAPH, true);
+ Assert.assertEquals(true, checkResults(EXPECTED_PATH));
+ }
+
+ private boolean checkResults(String expectedPath) throws Exception {
+ File dumped = null;
+ String format = conf.get(GenomixJob.OUTPUT_FORMAT);
+ if ("text".equalsIgnoreCase(format)) {
+ FileUtil.copyMerge(FileSystem.get(conf), new Path(HDFS_OUTPUT_PATH),
+ FileSystem.getLocal(new Configuration()), new Path(DUMPED_RESULT), false, conf, null);
+ dumped = new File(DUMPED_RESULT);
+ } else {
+
+ FileSystem.getLocal(new Configuration()).mkdirs(new Path(ACTUAL_RESULT_DIR + HDFS_OUTPUT_PATH));
+ File filePathTo = new File(CONVERT_RESULT);
+ BufferedWriter bw = new BufferedWriter(new FileWriter(filePathTo));
+ for (int i = 0; i < numPartitionPerMachine * numberOfNC; i++) {
+ String partname = "/part-" + i;
+ FileUtil.copy(FileSystem.get(conf), new Path(HDFS_OUTPUT_PATH
+ + partname), FileSystem.getLocal(new Configuration()),
+ new Path(ACTUAL_RESULT_DIR + HDFS_OUTPUT_PATH + partname), false, conf);
+
+ Path path = new Path(HDFS_OUTPUT_PATH + partname);
+ FileSystem dfs = FileSystem.get(conf);
+ if (dfs.getFileStatus(path).getLen() == 0) {
+ continue;
+ }
+ SequenceFile.Reader reader = new SequenceFile.Reader(dfs, path, conf);
+ KmerBytesWritable key = new KmerBytesWritable(conf.getInt(GenomixJob.KMER_LENGTH,
+ GenomixJob.DEFAULT_KMER));
+ KmerCountValue value = (KmerCountValue) ReflectionUtils.newInstance(reader.getValueClass(), conf);
+
+ while (reader.next(key, value)) {
+ if (key == null || value == null) {
+ break;
+ }
+ bw.write(key.toString() + "\t" + value.toString());
+ System.out.println(key.toString() + "\t" + value.toString());
+ bw.newLine();
+ }
+ reader.close();
+ }
+ bw.close();
+ dumped = new File(CONVERT_RESULT);
+ }
+
+ //TestUtils.compareWithSortedResult(new File(expectedPath), dumped);
+ return true;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ edu.uci.ics.hyracks.hdfs.utils.HyracksUtils.deinit();
+ cleanupHDFS();
+ }
+
+ private void cleanupHDFS() throws Exception {
+ dfsCluster.shutdown();
+ }
+}
diff --git a/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/MergePathTest.java b/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/MergePathTest.java
index 8313ef3..353fc6d 100644
--- a/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/MergePathTest.java
+++ b/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/MergePathTest.java
@@ -19,8 +19,8 @@
public class MergePathTest {
public static final String PATH_TO_TESTSTORE = "testcase/pathmerge/";
- public static final String NAIVE_DATA_INPUT = "genomix_result/pathmerge/final_naive_infinite";
- public static final String LOG_DATA_INPUT = "genomix_result/pathmerge/final_log_infinite";
+ public static final String NAIVE_DATA_INPUT = "genomix_result/pathmerge/naive_35";
+ public static final String LOG_DATA_INPUT = "genomix_result/pathmerge/log_12";
public static final String TEXT_OUTPUT = PATH_TO_TESTSTORE + "textfile";
public static final String CHAIN_OUTPUT = PATH_TO_TESTSTORE + "chain";
@@ -38,8 +38,8 @@
FileUtils.cleanDirectory(new File(CHAIN_OUTPUT));
generateTextFromPathmergeResult(NAIVE_DATA_INPUT, TEXT_OUTPUT, "/naive");
generateTextFromPathmergeResult(LOG_DATA_INPUT, TEXT_OUTPUT, "/log");
- generateSpecificLengthChainFromNaivePathmergeResult(NAIVE_DATA_INPUT, CHAIN_OUTPUT, maxLength);
- generateSpecificLengthChainFromLogPathmergeResult(LOG_DATA_INPUT, CHAIN_OUTPUT, maxLength);
+ //generateSpecificLengthChainFromNaivePathmergeResult(NAIVE_DATA_INPUT, CHAIN_OUTPUT, maxLength);
+ //generateSpecificLengthChainFromLogPathmergeResult(LOG_DATA_INPUT, CHAIN_OUTPUT, maxLength);
}
public static void generateTextFromPathmergeResult(String input, String outputDir, String fileName) throws IOException{
@@ -56,7 +56,8 @@
if (key == null || value == null){
break;
}
- if(value.getLengthOfMergeChain() != -1){
+ if(value.getLengthOfMergeChain() != -1
+ && value.getState() == State.FINAL_VERTEX){
bw.write(value.getLengthOfMergeChain() + "\t" +
value.getMergeChain().toString() + "\t" +
GeneCode.getSymbolFromBitMap(value.getAdjMap()));
@@ -99,7 +100,7 @@
BufferedWriter bw = new BufferedWriter(new FileWriter(new File(output + "/log")));
Configuration conf = new Configuration();
FileSystem fileSys = FileSystem.get(conf);
- for(int i = 0; i < 2; i++){
+ for(int i = 0; i < nc; i++){
Path path = new Path(input + "/part-" + i);
SequenceFile.Reader reader = new SequenceFile.Reader(fileSys, path, conf);
KmerBytesWritable key = new KmerBytesWritable(kmerSize);
diff --git a/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/PathMergeSmallTestCase.java b/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/PathMergeSmallTestCase.java
new file mode 100644
index 0000000..8191496
--- /dev/null
+++ b/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/PathMergeSmallTestCase.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.genomix.pregelix.pathmerge;
+
+import java.io.File;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.junit.Test;
+
+import edu.uci.ics.genomix.pregelix.example.util.TestUtils;
+import edu.uci.ics.pregelix.api.job.PregelixJob;
+import edu.uci.ics.pregelix.core.base.IDriver.Plan;
+import edu.uci.ics.pregelix.core.driver.Driver;
+import edu.uci.ics.pregelix.core.util.PregelixHyracksIntegrationUtil;
+
+public class PathMergeSmallTestCase extends TestCase {
+ private static String HDFS_INPUTPATH = "/BridgePath";
+ private static String HDFS_OUTPUTPAH = "/resultBridgePath";
+
+ /*private static String HDFS_INPUTPATH2 = "/CyclePath";
+ private static String HDFS_OUTPUTPAH2 = "/resultCyclePath";
+
+ private static String HDFS_INPUTPATH3 = "/LongPath";
+ private static String HDFS_OUTPUTPAH3 = "/resultLongPath";
+
+ private static String HDFS_INPUTPATH4 = "/Path";
+ private static String HDFS_OUTPUTPAH4 = "/resultPath";
+
+ private static String HDFS_INPUTPATH5 = "/SimplePath";
+ private static String HDFS_OUTPUTPAH5 = "/resultSimplePath";
+
+ private static String HDFS_INPUTPATH6 = "/SinglePath";
+ private static String HDFS_OUTPUTPAH6 = "/resultSinglePath";
+
+ private static String HDFS_INPUTPATH7 = "/TreePath";
+ private static String HDFS_OUTPUTPAH7 = "/resultTreePath";*/
+
+ private final PregelixJob job;
+ private final String resultFileDir;
+ private final String jobFile;
+ private final Driver driver = new Driver(this.getClass());
+ private final FileSystem dfs;
+
+ public PathMergeSmallTestCase(String hadoopConfPath, String jobName, String jobFile, String resultFile,
+ FileSystem dfs) throws Exception {
+ super("test");
+ this.jobFile = jobFile;
+ this.job = new PregelixJob("test");
+ this.job.getConfiguration().addResource(new Path(jobFile));
+ this.job.getConfiguration().addResource(new Path(hadoopConfPath));
+ Path[] inputPaths = FileInputFormat.getInputPaths(job);
+ if (inputPaths[0].toString().endsWith(HDFS_INPUTPATH)) {
+ FileInputFormat.setInputPaths(job, HDFS_INPUTPATH);
+ FileOutputFormat.setOutputPath(job, new Path(HDFS_OUTPUTPAH));
+ }
+ /*else if (inputPaths[0].toString().endsWith(HDFS_INPUTPATH2)) {
+ FileInputFormat.setInputPaths(job, HDFS_INPUTPATH2);
+ FileOutputFormat.setOutputPath(job, new Path(HDFS_OUTPUTPAH2));
+ } else if (inputPaths[0].toString().endsWith(HDFS_INPUTPATH3)) {
+ FileInputFormat.setInputPaths(job, HDFS_INPUTPATH3);
+ FileOutputFormat.setOutputPath(job, new Path(HDFS_OUTPUTPAH3));
+ } else if (inputPaths[0].toString().endsWith(HDFS_INPUTPATH4)) {
+ FileInputFormat.setInputPaths(job, HDFS_INPUTPATH4);
+ FileOutputFormat.setOutputPath(job, new Path(HDFS_OUTPUTPAH4));
+ } else if (inputPaths[0].toString().endsWith(HDFS_INPUTPATH5)) {
+ FileInputFormat.setInputPaths(job, HDFS_INPUTPATH5);
+ FileOutputFormat.setOutputPath(job, new Path(HDFS_OUTPUTPAH5));
+ } else if (inputPaths[0].toString().endsWith(HDFS_INPUTPATH6)) {
+ FileInputFormat.setInputPaths(job, HDFS_INPUTPATH6);
+ FileOutputFormat.setOutputPath(job, new Path(HDFS_OUTPUTPAH6));
+ } else if (inputPaths[0].toString().endsWith(HDFS_INPUTPATH7)) {
+ FileInputFormat.setInputPaths(job, HDFS_INPUTPATH7);
+ FileOutputFormat.setOutputPath(job, new Path(HDFS_OUTPUTPAH7));
+ }*/
+ job.setJobName(jobName);
+ this.resultFileDir = resultFile;
+ this.dfs = dfs;
+ }
+
+ private void waitawhile() throws InterruptedException {
+ synchronized (this) {
+ this.wait(20);
+ }
+ }
+
+ @Test
+ public void test() throws Exception {
+ setUp();
+ Plan[] plans = new Plan[] { Plan.OUTER_JOIN };
+ for (Plan plan : plans) {
+ driver.runJob(job, plan, PregelixHyracksIntegrationUtil.CC_HOST,
+ PregelixHyracksIntegrationUtil.TEST_HYRACKS_CC_CLIENT_PORT, false);
+ }
+ compareResults();
+ tearDown();
+ waitawhile();
+ }
+
+ private void compareResults() throws Exception {
+ dfs.copyToLocalFile(FileOutputFormat.getOutputPath(job), new Path(resultFileDir));
+ //TestUtils.compareWithResultDir(new File(expectedFileDir), new File(resultFileDir));
+ }
+
+ public String toString() {
+ return jobFile;
+ }
+}
diff --git a/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/PathMergeSmallTestSuite.java b/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/PathMergeSmallTestSuite.java
new file mode 100644
index 0000000..5329515
--- /dev/null
+++ b/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/PathMergeSmallTestSuite.java
@@ -0,0 +1,235 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.genomix.pregelix.pathmerge;
+
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+
+import junit.framework.Test;
+import junit.framework.TestResult;
+import junit.framework.TestSuite;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.mapred.JobConf;
+
+import edu.uci.ics.pregelix.core.jobgen.clusterconfig.ClusterConfig;
+import edu.uci.ics.pregelix.core.util.PregelixHyracksIntegrationUtil;
+
+@SuppressWarnings("deprecation")
+public class PathMergeSmallTestSuite extends TestSuite {
+ private static final Logger LOGGER = Logger.getLogger(PathMergeSmallTestSuite.class.getName());
+
+ private static final String ACTUAL_RESULT_DIR = "actual";
+ private static final String PATH_TO_HADOOP_CONF = "src/test/resources/hadoop/conf";
+ private static final String PATH_TO_CLUSTER_STORE = "src/test/resources/cluster/stores.properties";
+ private static final String PATH_TO_CLUSTER_PROPERTIES = "src/test/resources/cluster/cluster.properties";
+ private static final String PATH_TO_JOBS = "src/test/resources/jobs/";
+ private static final String PATH_TO_IGNORE = "src/test/resources/ignore.txt";
+ private static final String PATH_TO_ONLY = "src/test/resources/only.txt";
+
+ private static final String DATA_PATH = "data/input/BridgePath";
+ private static final String HDFS_PATH = "/BridgePath/";
+
+ /*private static final String DATA_PATH2 = "data/sequencefile/CyclePath";
+ private static final String HDFS_PATH2 = "/CyclePath/";
+
+ private static final String DATA_PATH3 = "data/sequencefile/LongPath";
+ private static final String HDFS_PATH3 = "/LongPath/";
+
+ private static final String DATA_PATH4 = "data/sequencefile/Path";
+ private static final String HDFS_PATH4 = "/Path/";
+
+ private static final String DATA_PATH5 = "data/sequencefile/SimplePath";
+ private static final String HDFS_PATH5 = "/SimplePath/";
+
+ private static final String DATA_PATH6 = "data/sequencefile/SinglePath";
+ private static final String HDFS_PATH6 = "/SinglePath/";
+
+ private static final String DATA_PATH7 = "data/sequencefile/TreePath";
+ private static final String HDFS_PATH7 = "/TreePath/";*/
+
+ private static final String HADOOP_CONF_PATH = ACTUAL_RESULT_DIR + File.separator + "conf.xml";
+ private MiniDFSCluster dfsCluster;
+
+ private JobConf conf = new JobConf();
+ private int numberOfNC = 1;
+
+ public void setUp() throws Exception {
+ ClusterConfig.setStorePath(PATH_TO_CLUSTER_STORE);
+ ClusterConfig.setClusterPropertiesPath(PATH_TO_CLUSTER_PROPERTIES);
+ cleanupStores();
+ PregelixHyracksIntegrationUtil.init("src/test/resources/topology.xml");
+ LOGGER.info("Hyracks mini-cluster started");
+ FileUtils.forceMkdir(new File(ACTUAL_RESULT_DIR));
+ FileUtils.cleanDirectory(new File(ACTUAL_RESULT_DIR));
+ startHDFS();
+ }
+
+ private void cleanupStores() throws IOException {
+ FileUtils.forceMkdir(new File("teststore"));
+ FileUtils.forceMkdir(new File("build"));
+ FileUtils.cleanDirectory(new File("teststore"));
+ FileUtils.cleanDirectory(new File("build"));
+ }
+
+ private void startHDFS() throws IOException {
+ conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml"));
+ conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
+ conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));
+ FileSystem lfs = FileSystem.getLocal(new Configuration());
+ lfs.delete(new Path("build"), true);
+ System.setProperty("hadoop.log.dir", "logs");
+ dfsCluster = new MiniDFSCluster(conf, numberOfNC, true, null);
+ FileSystem dfs = FileSystem.get(conf);
+ Path src = new Path(DATA_PATH);
+ Path dest = new Path(HDFS_PATH);
+ dfs.mkdirs(dest);
+ dfs.copyFromLocalFile(src, dest);
+
+ /*src = new Path(DATA_PATH2);
+ dest = new Path(HDFS_PATH2);
+ dfs.mkdirs(dest);
+ dfs.copyFromLocalFile(src, dest);
+
+ src = new Path(DATA_PATH3);
+ dest = new Path(HDFS_PATH3);
+ dfs.mkdirs(dest);
+ dfs.copyFromLocalFile(src, dest);
+
+ src = new Path(DATA_PATH4);
+ dest = new Path(HDFS_PATH4);
+ dfs.mkdirs(dest);
+ dfs.copyFromLocalFile(src, dest);
+
+ src = new Path(DATA_PATH5);
+ dest = new Path(HDFS_PATH5);
+ dfs.mkdirs(dest);
+ dfs.copyFromLocalFile(src, dest);
+
+ src = new Path(DATA_PATH6);
+ dest = new Path(HDFS_PATH6);
+ dfs.mkdirs(dest);
+ dfs.copyFromLocalFile(src, dest);
+
+ src = new Path(DATA_PATH7);
+ dest = new Path(HDFS_PATH7);
+ dfs.mkdirs(dest);
+ dfs.copyFromLocalFile(src, dest);*/
+
+ DataOutputStream confOutput = new DataOutputStream(new FileOutputStream(new File(HADOOP_CONF_PATH)));
+ conf.writeXml(confOutput);
+ confOutput.flush();
+ confOutput.close();
+ }
+
+ /**
+ * cleanup hdfs cluster
+ */
+ private void cleanupHDFS() throws Exception {
+ dfsCluster.shutdown();
+ }
+
+ public void tearDown() throws Exception {
+ PregelixHyracksIntegrationUtil.deinit();
+ LOGGER.info("Hyracks mini-cluster shut down");
+ cleanupHDFS();
+ }
+
+ public static Test suite() throws Exception {
+ List<String> ignores = getFileList(PATH_TO_IGNORE);
+ List<String> onlys = getFileList(PATH_TO_ONLY);
+ File testData = new File(PATH_TO_JOBS);
+ File[] queries = testData.listFiles();
+ PathMergeSmallTestSuite testSuite = new PathMergeSmallTestSuite();
+ testSuite.setUp();
+ boolean onlyEnabled = false;
+ FileSystem dfs = FileSystem.get(testSuite.conf);
+
+ if (onlys.size() > 0) {
+ onlyEnabled = true;
+ }
+ for (File qFile : queries) {
+ if (isInList(ignores, qFile.getName()))
+ continue;
+
+ if (qFile.isFile()) {
+ if (onlyEnabled && !isInList(onlys, qFile.getName())) {
+ continue;
+ } else {
+ String resultFileName = ACTUAL_RESULT_DIR + File.separator + jobExtToResExt(qFile.getName());
+ testSuite.addTest(new PathMergeSmallTestCase(HADOOP_CONF_PATH, qFile.getName(), qFile.getAbsolutePath()
+ .toString(), resultFileName, dfs));
+ }
+ }
+ }
+ return testSuite;
+ }
+
+ /**
+ * Runs the tests and collects their result in a TestResult.
+ */
+ @Override
+ public void run(TestResult result) {
+ try {
+ int testCount = countTestCases();
+ for (int i = 0; i < testCount; i++) {
+ // cleanupStores();
+ Test each = this.testAt(i);
+ if (result.shouldStop())
+ break;
+ runTest(each, result);
+ }
+ tearDown();
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ protected static List<String> getFileList(String ignorePath) throws FileNotFoundException, IOException {
+ BufferedReader reader = new BufferedReader(new FileReader(ignorePath));
+ String s = null;
+ List<String> ignores = new ArrayList<String>();
+ while ((s = reader.readLine()) != null) {
+ ignores.add(s);
+ }
+ reader.close();
+ return ignores;
+ }
+
+ private static String jobExtToResExt(String fname) {
+ int dot = fname.lastIndexOf('.');
+ return fname.substring(0, dot);
+ }
+
+ private static boolean isInList(List<String> onlys, String name) {
+ for (String only : onlys)
+ if (name.indexOf(only) >= 0)
+ return true;
+ return false;
+ }
+
+}
diff --git a/genomix/genomix-pregelix/src/test/resources/cluster/cluster.properties b/genomix/genomix-pregelix/src/test/resources/cluster/cluster.properties
index 14f8bd4..d0722ce 100644
--- a/genomix/genomix-pregelix/src/test/resources/cluster/cluster.properties
+++ b/genomix/genomix-pregelix/src/test/resources/cluster/cluster.properties
@@ -20,7 +20,7 @@
NCLOGS_DIR=$NCTMP_DIR/logs
#Comma separated I/O directories for the spilling of external sort
-IO_DIRS="/tmp/t3,/tmp/t4"
+IO_DIRS="/tmp/t3"
#The JAVA_HOME
JAVA_HOME=$JAVA_HOME