new graph building progress
diff --git a/genomix/genomix-hadoop/expected/result1 b/genomix/genomix-hadoop/expected/result1
deleted file mode 100644
index ba52008..0000000
--- a/genomix/genomix-hadoop/expected/result1
+++ /dev/null
@@ -1,8 +0,0 @@
-GCA ACT|T 3
-AGC |A 1
-CGC T|AT 2
-TGC |A 1
-ATC C|G 2
-TCG A|C 2
-CAT G|C 2
-GCT C| 1
diff --git a/genomix/genomix-hadoop/expected/result2 b/genomix/genomix-hadoop/expected/result2
deleted file mode 100755
index db55a38..0000000
--- a/genomix/genomix-hadoop/expected/result2
+++ /dev/null
@@ -1,8 +0,0 @@
-GCA -72
-AGC 1
-CGC -119
-TGC 1
-ATC 36
-TCG 18
-CAT 66
-GCT 32
diff --git a/genomix/genomix-hadoop/expected/result3 b/genomix/genomix-hadoop/expected/result3
deleted file mode 100644
index 0c95da0..0000000
--- a/genomix/genomix-hadoop/expected/result3
+++ /dev/null
@@ -1 +0,0 @@
-CATCG 66 -127
diff --git a/genomix/genomix-hadoop/expected/result4 b/genomix/genomix-hadoop/expected/result4
deleted file mode 100644
index 0c95da0..0000000
--- a/genomix/genomix-hadoop/expected/result4
+++ /dev/null
@@ -1 +0,0 @@
-CATCG 66 -127
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/genomix/hadoop/velvetgraphbuilding/GraphBuildingDriver.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/genomix/hadoop/velvetgraphbuilding/GraphBuildingDriver.java
index 68ce1c2..3bd1bf8 100644
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/genomix/hadoop/velvetgraphbuilding/GraphBuildingDriver.java
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/genomix/hadoop/velvetgraphbuilding/GraphBuildingDriver.java
@@ -51,7 +51,7 @@
if (onlyTest1stJob == true) {
runfirstjob(inputPath, numReducers, sizeKmer, seqOutput, defaultConfPath);
} else {
- runfirstjob(inputPath, numReducers, sizeKmer, false, defaultConfPath);
+ runfirstjob(inputPath, numReducers, sizeKmer, true, defaultConfPath);
runsecondjob(inputPath, outputPath, numReducers, sizeKmer, seqOutput, defaultConfPath);
}
}
@@ -100,8 +100,10 @@
conf.setMapperClass(DeepGraphBuildingMapper.class);
conf.setReducerClass(DeepGraphBuildingReducer.class);
+
conf.setMapOutputKeyClass(PositionWritable.class);
conf.setMapOutputValueClass(PositionListAndKmerWritable.class);
+
conf.setPartitionerClass(ReadIDPartitioner.class);
@@ -113,13 +115,18 @@
conf.setOutputFormat(SequenceFileOutputFormat.class);
else
conf.setOutputFormat(TextOutputFormat.class);
-
+ if(numReducers != 0){
conf.setOutputKeyClass(NodeWritable.class);
conf.setOutputValueClass(NullWritable.class);
+ }
+ else {
+ conf.setOutputKeyClass(PositionWritable.class);
+ conf.setOutputValueClass(PositionListAndKmerWritable.class);
+ }
FileInputFormat.setInputPaths(conf, new Path(inputPath + "-step1"));
FileOutputFormat.setOutputPath(conf, new Path(outputPath));
- conf.setNumReduceTasks(1);
+ conf.setNumReduceTasks(numReducers);
FileSystem dfs = FileSystem.get(conf);
dfs.delete(new Path(outputPath), true);
JobClient.runJob(conf);
diff --git a/genomix/genomix-hadoop/src/test/java/edu/uci/ics/genomix/hadoop/velvetgraphbuilding/NewGraphBuildingTest.java b/genomix/genomix-hadoop/src/test/java/edu/uci/ics/genomix/hadoop/velvetgraphbuilding/NewGraphBuildingTest.java
index 18f95b3..cc24133 100644
--- a/genomix/genomix-hadoop/src/test/java/edu/uci/ics/genomix/hadoop/velvetgraphbuilding/NewGraphBuildingTest.java
+++ b/genomix/genomix-hadoop/src/test/java/edu/uci/ics/genomix/hadoop/velvetgraphbuilding/NewGraphBuildingTest.java
@@ -17,21 +17,21 @@
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.util.ReflectionUtils;
import org.junit.Test;
+
+import edu.uci.ics.genomix.hadoop.utils.TestUtils;
@SuppressWarnings("deprecation")
public class NewGraphBuildingTest {
private JobConf conf = new JobConf();
private static final String ACTUAL_RESULT_DIR = "actual1";
- private static final String COMPARE_DIR = "compare";
private static final String HADOOP_CONF_PATH = ACTUAL_RESULT_DIR + File.separator + "conf.xml";
- private static final String DATA_PATH = "data/webmap/Test.txt";
+ private static final String DATA_PATH = "data/webmap/text.txt";
private static final String HDFS_PATH = "/webmap";
private static final String RESULT_PATH = "/result1";
- private static final String EXPECTED_PATH = "expected/result1";
- private static final String TEST_SOURCE_DIR = COMPARE_DIR + RESULT_PATH;
- private static final int COUNT_REDUCER = 4;
- private static final int SIZE_KMER = 3;
+ private static final String EXPECTED_PATH = "expected/result_after_kmerAggregate";
+ private static final int COUNT_REDUCER = 0;
+ private static final int SIZE_KMER = 5;
private static final String GRAPHVIZ = "Graphviz";
private MiniDFSCluster dfsCluster;
@@ -41,8 +41,11 @@
@SuppressWarnings("resource")
@Test
public void test() throws Exception {
+ FileUtils.forceMkdir(new File(ACTUAL_RESULT_DIR));
+ FileUtils.cleanDirectory(new File(ACTUAL_RESULT_DIR));
startHadoop();
- TestGroupbyKmer();
+// TestGroupbyKmer();
+ TestMapKmerToRead();
/* SequenceFile.Reader reader = null;
Path path = new Path(RESULT_PATH + "/part-00000");
@@ -83,27 +86,27 @@
bw2.close();
bw.close();*/
-
-// TestUtils.compareWithResult(new File(TEST_SOURCE_DIR + "/comparesource.txt"), new File(EXPECTED_PATH));
cleanupHadoop();
}
public void TestGroupbyKmer() throws Exception {
- FileUtils.forceMkdir(new File(ACTUAL_RESULT_DIR));
- FileUtils.cleanDirectory(new File(ACTUAL_RESULT_DIR));
GraphBuildingDriver tldriver = new GraphBuildingDriver();
tldriver.run(HDFS_PATH, RESULT_PATH, COUNT_REDUCER, SIZE_KMER, true, false, HADOOP_CONF_PATH);
- dumpResult();
+ dumpGroupByKmerResult();
+ TestUtils.compareWithResult(new File(ACTUAL_RESULT_DIR + HDFS_PATH + "-step1" + "/part-00000"), new File(EXPECTED_PATH));
}
public void TestMapKmerToRead() throws Exception {
-
+ GraphBuildingDriver tldriver = new GraphBuildingDriver();
+ tldriver.run(HDFS_PATH, RESULT_PATH, COUNT_REDUCER, SIZE_KMER, false, false, HADOOP_CONF_PATH);
+ dumpResult();
}
public void TestGroupByReadID() throws Exception {
}
+
private void startHadoop() throws IOException {
FileSystem lfs = FileSystem.getLocal(new Configuration());
lfs.delete(new Path("build"), true);
@@ -128,6 +131,12 @@
dfsCluster.shutdown();
}
+ private void dumpGroupByKmerResult() throws IOException {
+ Path src = new Path(HDFS_PATH + "-step1");
+ Path dest = new Path(ACTUAL_RESULT_DIR);
+ dfs.copyToLocalFile(src, dest);
+ }
+
private void dumpResult() throws IOException {
Path src = new Path(RESULT_PATH);
Path dest = new Path(ACTUAL_RESULT_DIR);