h1 changes
diff --git a/genomix/genomix-hadoop/actual3/complete2/.complete2-r-00000.crc b/genomix/genomix-hadoop/actual3/complete2/.complete2-r-00000.crc
index b0b2753..bccc1b3 100644
--- a/genomix/genomix-hadoop/actual3/complete2/.complete2-r-00000.crc
+++ b/genomix/genomix-hadoop/actual3/complete2/.complete2-r-00000.crc
Binary files differ
diff --git a/genomix/genomix-hadoop/actual3/complete2/complete2-r-00000 b/genomix/genomix-hadoop/actual3/complete2/complete2-r-00000
index d3d3667..2704be9 100755
--- a/genomix/genomix-hadoop/actual3/complete2/complete2-r-00000
+++ b/genomix/genomix-hadoop/actual3/complete2/complete2-r-00000
Binary files differ
diff --git a/genomix/genomix-hadoop/actual3/conf.xml b/genomix/genomix-hadoop/actual3/conf.xml
index 16a0edc..f662215 100644
--- a/genomix/genomix-hadoop/actual3/conf.xml
+++ b/genomix/genomix-hadoop/actual3/conf.xml
@@ -12,7 +12,7 @@
<property><name>dfs.namenode.logging.level</name><value>info</value></property>
<property><name>dfs.datanode.address</name><value>127.0.0.1:0</value></property>
<property><name>io.skip.checksum.errors</name><value>false</value></property>
-<property><name>fs.default.name</name><value>hdfs://localhost:62106</value></property>
+<property><name>fs.default.name</name><value>hdfs://localhost:50310</value></property>
<property><name>mapred.child.tmp</name><value>./tmp</value></property>
<property><name>fs.har.impl.disable.cache</name><value>true</value></property>
<property><name>dfs.safemode.threshold.pct</name><value>0.999f</value></property>
@@ -125,7 +125,7 @@
<property><name>mapred.map.output.compression.codec</name><value>org.apache.hadoop.io.compress.DefaultCodec</value></property>
<property><name>mapred.tasktracker.dns.interface</name><value>default</value></property>
<property><name>dfs.namenode.decommission.interval</name><value>3</value></property>
-<property><name>dfs.http.address</name><value>localhost:62107</value></property>
+<property><name>dfs.http.address</name><value>localhost:50311</value></property>
<property><name>dfs.heartbeat.interval</name><value>3</value></property>
<property><name>mapred.job.tracker</name><value>local</value></property>
<property><name>io.seqfile.sorter.recordlimit</name><value>1000000</value></property>
diff --git a/genomix/genomix-hadoop/pom.xml b/genomix/genomix-hadoop/pom.xml
index f791f5b..ff6a7ee 100755
--- a/genomix/genomix-hadoop/pom.xml
+++ b/genomix/genomix-hadoop/pom.xml
@@ -22,8 +22,9 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>2.0.2</version>
<configuration>
- <source>1.6</source>
- <target>1.6</target>
+ <source>1.7</source>
+ <target>1.7</target>
+ <fork>true</fork>
</configuration>
</plugin>
<plugin>
diff --git a/genomix/genomix-hadoop/src/.DS_Store b/genomix/genomix-hadoop/src/.DS_Store
new file mode 100644
index 0000000..1cf57ac
--- /dev/null
+++ b/genomix/genomix-hadoop/src/.DS_Store
Binary files differ
diff --git a/genomix/genomix-hadoop/src/main/.DS_Store b/genomix/genomix-hadoop/src/main/.DS_Store
new file mode 100644
index 0000000..f2a5476
--- /dev/null
+++ b/genomix/genomix-hadoop/src/main/.DS_Store
Binary files differ
diff --git a/genomix/genomix-hadoop/src/main/java/.DS_Store b/genomix/genomix-hadoop/src/main/java/.DS_Store
new file mode 100644
index 0000000..7806fed
--- /dev/null
+++ b/genomix/genomix-hadoop/src/main/java/.DS_Store
Binary files differ
diff --git a/genomix/genomix-hadoop/src/main/java/edu/.DS_Store b/genomix/genomix-hadoop/src/main/java/edu/.DS_Store
new file mode 100644
index 0000000..bc05a64
--- /dev/null
+++ b/genomix/genomix-hadoop/src/main/java/edu/.DS_Store
Binary files differ
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/.DS_Store b/genomix/genomix-hadoop/src/main/java/edu/uci/.DS_Store
new file mode 100644
index 0000000..ec6ffce
--- /dev/null
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/.DS_Store
Binary files differ
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/.DS_Store b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/.DS_Store
new file mode 100644
index 0000000..22fef5d
--- /dev/null
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/.DS_Store
Binary files differ
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/MergePathDriver.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/MergePathDriver.java
index cfdf8d5..99eec46 100644
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/MergePathDriver.java
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/MergePathDriver.java
@@ -23,8 +23,10 @@
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
+import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.mapred.lib.MultipleOutputs;
import org.apache.hadoop.mapred.lib.MultipleSequenceFileOutputFormat;
+import org.apache.hadoop.mapred.lib.MultipleTextOutputFormat;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
@@ -83,8 +85,9 @@
FileSystem dfs = FileSystem.get(conf);
dfs.delete(new Path(inputPath + "-step1"), true);
JobClient.runJob(conf);
+ int iMerge = 0;
/*----------------------------------------------------------------------*/
- for(int iMerge = 0; iMerge < mergeRound; iMerge ++){
+ for(iMerge = 0; iMerge < mergeRound; iMerge ++){
conf = new JobConf(MergePathDriver.class);
conf.setInt("sizeKmer", sizeKmer);
@@ -102,7 +105,6 @@
conf.setMapOutputValueClass(MergePathValueWritable.class);
conf.setInputFormat(SequenceFileInputFormat.class);
- conf.setOutputFormat(MultipleSequenceFileOutputFormat.class);
String uncomplete = "uncomplete" + iMerge;
String complete = "complete" + iMerge;
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/MergePathMapper.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/MergePathMapper.java
index c3255f2..ff4c2ce 100644
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/MergePathMapper.java
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/MergePathMapper.java
@@ -28,24 +28,24 @@
@SuppressWarnings("deprecation")
public class MergePathMapper extends MapReduceBase implements
- Mapper<KmerBytesWritable, MergePathValueWritable, KmerBytesWritable, MergePathValueWritable> {
+ Mapper<VKmerBytesWritable, MergePathValueWritable, VKmerBytesWritable, MergePathValueWritable> {
private int KMER_SIZE;
private VKmerBytesWritableFactory outputKmerFactory;
- private MergePathValueWritable outputAdjList;
+ private MergePathValueWritable outputValue;
private VKmerBytesWritable tmpKmer;
private VKmerBytesWritable outputKmer;
public void configure(JobConf job) {
KMER_SIZE = job.getInt("sizeKmer", 0);
outputKmerFactory = new VKmerBytesWritableFactory(KMER_SIZE);
- outputAdjList = new MergePathValueWritable();
+ outputValue = new MergePathValueWritable();
tmpKmer = new VKmerBytesWritable(KMER_SIZE);
outputKmer = new VKmerBytesWritable(KMER_SIZE);
}
@Override
- public void map(KmerBytesWritable key, MergePathValueWritable value,
- OutputCollector<KmerBytesWritable, MergePathValueWritable> output, Reporter reporter) throws IOException {
+ public void map(VKmerBytesWritable key, MergePathValueWritable value,
+ OutputCollector<VKmerBytesWritable, MergePathValueWritable> output, Reporter reporter) throws IOException {
byte precursor = (byte) 0xF0;
byte succeed = (byte) 0x0F;
@@ -59,14 +59,14 @@
byte succeedCode = GeneCode.getGeneCodeFromBitMap(succeed);
tmpKmer.set(outputKmerFactory.getLastKmerFromChain(KMER_SIZE, key));
outputKmer.set(outputKmerFactory.shiftKmerWithNextCode(tmpKmer, succeedCode));
-
- KmerBytesWritable mergedKmer = outputKmerFactory.getFirstKmerFromChain(value.getKmerSize()
- - (KMER_SIZE - 1), value.getKmer());
- outputAdjList.set(mergedKmer, adjBitMap, bitFlag);
- output.collect(outputKmer, outputAdjList);
+
+ tmpKmer.set(outputKmerFactory.getFirstKmerFromChain(key.getKmerLength() - (KMER_SIZE - 1), key));
+ outputValue.set(tmpKmer, adjBitMap, bitFlag);
+ output.collect(outputKmer, outputValue);
} else {
- outputAdjList.set(value);
- output.collect(key, outputAdjList);
+ outputKmer.set(key);
+ outputValue.set(value);
+ output.collect(key, outputValue);
}
}
}
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/MergePathReducer.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/MergePathReducer.java
index cead0e8..44f7e8e 100644
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/MergePathReducer.java
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/MergePathReducer.java
@@ -29,12 +29,14 @@
@SuppressWarnings("deprecation")
public class MergePathReducer extends MapReduceBase implements
- Reducer<KmerBytesWritable, MergePathValueWritable, KmerBytesWritable, MergePathValueWritable> {
+ Reducer<VKmerBytesWritable, MergePathValueWritable, VKmerBytesWritable, MergePathValueWritable> {
private VKmerBytesWritableFactory kmerFactory;
private VKmerBytesWritable outputKmer;
private VKmerBytesWritable tmpKmer;
private int KMER_SIZE;
- private MergePathValueWritable outputAdjList;
+ private MergePathValueWritable outputValue;
+ private MergePathValueWritable tmpOutputValue;
+
MultipleOutputs mos = null;
private int I_MERGE;
@@ -42,7 +44,7 @@
mos = new MultipleOutputs(job);
I_MERGE = Integer.parseInt(job.get("iMerge"));
KMER_SIZE = job.getInt("sizeKmer", 0);
- outputAdjList = new MergePathValueWritable();
+ outputValue = new MergePathValueWritable();
kmerFactory = new VKmerBytesWritableFactory(KMER_SIZE);
outputKmer = new VKmerBytesWritable(KMER_SIZE);
tmpKmer = new VKmerBytesWritable(KMER_SIZE);
@@ -50,58 +52,72 @@
@SuppressWarnings("unchecked")
@Override
- public void reduce(KmerBytesWritable key, Iterator<MergePathValueWritable> values,
- OutputCollector<KmerBytesWritable, MergePathValueWritable> output, Reporter reporter) throws IOException {
- outputAdjList = values.next();
-
-
+ public void reduce(VKmerBytesWritable key, Iterator<MergePathValueWritable> values,
+ OutputCollector<VKmerBytesWritable, MergePathValueWritable> output, Reporter reporter) throws IOException {
+ outputValue = values.next();
if (values.hasNext() == true) {
-
- if (outputAdjList.getFlag() == 1) {
- byte adjBitMap = outputAdjList.getAdjBitMap();
- byte bitFlag = outputAdjList.getFlag();
- outputKmer.set(kmerFactory.mergeTwoKmer(outputAdjList.getKmer(), key));
-
- outputAdjList = values.next();
- byte nextAdj = outputAdjList.getAdjBitMap();
+ if(outputValue.getFlag() != 1){
+ byte nextAdj = outputValue.getAdjBitMap();
byte succeed = (byte) 0x0F;
succeed = (byte) (succeed & nextAdj);
- adjBitMap = (byte) (adjBitMap & 0xF0);
- adjBitMap = (byte) (adjBitMap | succeed);
- outputAdjList.set(null, 0, 0, adjBitMap, bitFlag, KMER_SIZE + outputAdjList.getKmerSize());
-
- mos.getCollector("uncomplete" + I_MERGE, reporter).collect(outputKmer, outputAdjList);
- } else {
- byte nextAdj = outputAdjList.getAdjBitMap();
- byte succeed = (byte) 0x0F;
- succeed = (byte) (succeed & nextAdj);
- outputAdjList = values.next();
- byte adjBitMap = outputAdjList.getAdjBitMap();
- byte flag = outputAdjList.getFlag();
- int kmerSize = outputAdjList.getKmerSize();
- outputKmer.set(kmerFactory.mergeTwoKmer(outputAdjList.getKmer(), key));
+ outputValue = values.next();
+ byte adjBitMap = outputValue.getAdjBitMap();
+ byte flag = outputValue.getFlag();
+ outputKmer.set(kmerFactory.mergeTwoKmer(outputValue.getKmer(), key));
+
adjBitMap = (byte) (adjBitMap & 0xF0);
adjBitMap = (byte) (adjBitMap | succeed);
- outputAdjList.set(null, 0, 0, adjBitMap, flag, KMER_SIZE + kmerSize);
-
- mos.getCollector("uncomplete" + I_MERGE, reporter).collect(outputKmer, outputAdjList);
+ outputValue.set(null, 0, 0, adjBitMap, flag, 0);
+ mos.getCollector("uncomplete" + I_MERGE, reporter).collect(outputKmer, outputValue);
+ }
+ else{
+ tmpOutputValue.set(outputValue);
+ byte tmpAdjMap = tmpOutputValue.getAdjBitMap();
+
+ outputValue = values.next();
+ if(outputValue.getFlag() != 1) {
+ outputKmer.set(kmerFactory.mergeTwoKmer(tmpOutputValue.getKmer(), key));
+
+ byte nextAdj = outputValue.getAdjBitMap();
+ byte succeed = (byte) 0x0F;
+ succeed = (byte) (succeed & nextAdj);
+ tmpAdjMap = (byte) (tmpAdjMap & 0xF0);
+ tmpAdjMap = (byte) (tmpAdjMap | succeed);
+ outputValue.set(null, 0, 0, tmpAdjMap, tmpOutputValue.getFlag(), 0);
+ mos.getCollector("uncomplete" + I_MERGE, reporter).collect(outputKmer, outputValue);
+ }
+ else{
+
+ tmpKmer.set(kmerFactory.getFirstKmerFromChain(KMER_SIZE-1, key));
+ outputKmer.set(kmerFactory.mergeTwoKmer(tmpOutputValue.getKmer(), tmpKmer));
+ tmpOutputValue.set(null, 0, 0, tmpAdjMap, tmpOutputValue.getFlag(), 0);
+ mos.getCollector("complete" + I_MERGE, reporter).collect(outputKmer, tmpOutputValue);
+
+ tmpKmer.set(kmerFactory.getFirstKmerFromChain(KMER_SIZE-1, key));
+ outputKmer.set(kmerFactory.mergeTwoKmer(outputValue.getKmer(), tmpKmer));
+ outputValue.set(null, 0, 0, outputValue.getAdjBitMap(), outputValue.getFlag(), 0);
+ mos.getCollector("complete" + I_MERGE, reporter).collect(outputKmer, outputValue);
+
+ while(values.hasNext()) {
+ outputValue = values.next();
+ tmpKmer.set(kmerFactory.getFirstKmerFromChain(KMER_SIZE-1, key));
+ outputKmer.set(kmerFactory.mergeTwoKmer(outputValue.getKmer(), tmpKmer));
+ outputValue.set(null, 0, 0, outputValue.getAdjBitMap(), outputValue.getFlag(), 0);
+ mos.getCollector("complete" + I_MERGE, reporter).collect(outputKmer, outputValue);
+ }
+ }
}
} else {
- if (outputAdjList.getFlag() != 0) {
- byte adjBitMap = outputAdjList.getAdjBitMap();
- byte flag = outputAdjList.getFlag();
- int kmerSize = outputAdjList.getKmerSize();
-
- tmpKmer.set(kmerFactory.getFirstKmerFromChain(KMER_SIZE - 1, key));
- outputKmer.set(kmerFactory.mergeTwoKmer(outputAdjList.getKmer(), tmpKmer));
- outputAdjList.set(null, 0, 0, adjBitMap, flag, KMER_SIZE + kmerSize);
- mos.getCollector("complete" + I_MERGE, reporter).collect(outputKmer, outputAdjList);
+ if (outputValue.getFlag() != 0) {
+ tmpKmer.set(kmerFactory.getFirstKmerFromChain(KMER_SIZE-1, key));
+ outputKmer.set(kmerFactory.mergeTwoKmer(outputValue.getKmer(), tmpKmer));
+ outputValue.set(null, 0, 0, outputValue.getAdjBitMap(), outputValue.getFlag(), 0);
+ mos.getCollector("complete" + I_MERGE, reporter).collect(outputKmer, outputValue);
} else
- mos.getCollector("uncomplete" + I_MERGE, reporter).collect(key, outputAdjList);
+ mos.getCollector("uncomplete" + I_MERGE, reporter).collect(key, outputValue);
}
}
-
public void close() throws IOException {
// TODO Auto-generated method stub
mos.close();
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/SNodeInitialMapper.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/SNodeInitialMapper.java
index ae824e7..5c4bdbe 100644
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/SNodeInitialMapper.java
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/SNodeInitialMapper.java
@@ -22,8 +22,8 @@
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-
import edu.uci.ics.genomix.type.KmerBytesWritable;
+import edu.uci.ics.genomix.type.GeneCode;
@SuppressWarnings("deprecation")
public class SNodeInitialMapper extends MapReduceBase implements
@@ -101,39 +101,34 @@
byte precursor = (byte) 0xF0;
byte succeed = (byte) 0x0F;
byte adjBitMap = value.get();
- byte flag = (byte) 0;
+ byte bitFlag = (byte) 0;
precursor = (byte) (precursor & adjBitMap);
precursor = (byte) ((precursor & 0xff) >> 4);
succeed = (byte) (succeed & adjBitMap);
boolean inDegree = measureDegree(precursor);
boolean outDegree = measureDegree(succeed);
- byte initial = 0;
- if (inDegree == true && outDegree == false) {
- flag = (byte) 2;
- switch (succeed) {
- case 1:
- initial = (byte) 0x00;
- break;
- case 2:
- initial = (byte) 0x01;
- break;
- case 4:
- initial = (byte) 0x02;
- break;
- case 8:
- initial = (byte) 0x03;
- break;
- }
- outputKmer.set(key);
- outputKmer.shiftKmerWithNextCode(initial);
- adjBitMap = (byte) (adjBitMap & 0xF0);
- outputAdjList.set(null, 0, 0, adjBitMap, flag, KMER_SIZE);
- output.collect(outputKmer, outputAdjList);
- }
+
if (inDegree == false && outDegree == false) {
outputKmer.set(key);
- outputAdjList.set(null, 0, 0, adjBitMap, flag, KMER_SIZE);
+ bitFlag = (byte) 2;
+ outputAdjList.set(null, 0, 0, adjBitMap, bitFlag, KMER_SIZE);
output.collect(outputKmer, outputAdjList);
}
+ else{
+ for(int i = 0 ; i < 4; i ++){
+ byte temp = 0x01;
+ byte shiftedCode = 0;
+ temp = (byte)(temp << i);
+ temp = (byte) (succeed & temp);
+ if(temp != 0 ){
+ byte succeedCode = GeneCode.getGeneCodeFromBitMap(temp);
+ shiftedCode = key.shiftKmerWithNextCode(succeedCode);
+ outputKmer.set(key);
+ outputAdjList.set(null, 0, 0, (byte)0, bitFlag, KMER_SIZE);
+ output.collect(outputKmer, outputAdjList);
+ key.shiftKmerWithPreCode(shiftedCode);
+ }
+ }
+ }
}
}
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/SNodeInitialReducer.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/SNodeInitialReducer.java
index 734abd6..42badf5 100644
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/SNodeInitialReducer.java
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmerging/SNodeInitialReducer.java
@@ -16,50 +16,50 @@
import java.io.IOException;
import java.util.Iterator;
-
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-
import edu.uci.ics.genomix.type.KmerBytesWritable;
+import edu.uci.ics.genomix.type.VKmerBytesWritable;
@SuppressWarnings("deprecation")
public class SNodeInitialReducer extends MapReduceBase implements
- Reducer<KmerBytesWritable, MergePathValueWritable, KmerBytesWritable, MergePathValueWritable> {
- private MergePathValueWritable outputAdjList = new MergePathValueWritable();
+ Reducer<KmerBytesWritable, MergePathValueWritable, VKmerBytesWritable, MergePathValueWritable> {
+ private VKmerBytesWritable outputKmer = new VKmerBytesWritable();
+ private MergePathValueWritable outputValue = new MergePathValueWritable();
@Override
public void reduce(KmerBytesWritable key, Iterator<MergePathValueWritable> values,
- OutputCollector<KmerBytesWritable, MergePathValueWritable> output, Reporter reporter) throws IOException {
- outputAdjList = values.next();
+ OutputCollector<VKmerBytesWritable, MergePathValueWritable> output, Reporter reporter) throws IOException {
+ outputKmer.set(key);
+ outputValue = values.next();
if (values.hasNext() == true) {
- if (outputAdjList.getFlag() != 2) {
- byte adjBitMap = outputAdjList.getAdjBitMap();
- int kmerSize = outputAdjList.getKmerSize();
+ if (outputValue.getFlag() == 2) {
byte bitFlag = 1;
- outputAdjList.set(null, 0, 0, adjBitMap, bitFlag, kmerSize);
- output.collect(key, outputAdjList);
-
+ outputValue.set(null, 0, 0, outputValue.getAdjBitMap(), bitFlag, outputValue.getKmerLength());
+ output.collect(outputKmer, outputValue);
} else {
boolean flag = false;
while (values.hasNext()) {
- outputAdjList = values.next();
- if (outputAdjList.getFlag() != 2) {
+ outputValue = values.next();
+ if (outputValue.getFlag() == 2) {
flag = true;
break;
}
}
if (flag == true) {
- byte adjBitMap = outputAdjList.getAdjBitMap();
- int kmerSize = outputAdjList.getKmerSize();
byte bitFlag = 1;
- outputAdjList.set(null, 0, 0, adjBitMap, bitFlag, kmerSize);
- output.collect(key, outputAdjList);
+ outputValue.set(null, 0, 0, outputValue.getAdjBitMap(), bitFlag, outputValue.getKmerLength());
+ output.collect(outputKmer, outputValue);
}
}
} else {
- output.collect(key, outputAdjList);
+ if (outputValue.getFlag() == 2) {
+ byte bitFlag = 0;
+ outputValue.set(null, 0, 0, outputValue.getAdjBitMap(), bitFlag, outputValue.getKmerLength());
+ output.collect(outputKmer, outputValue);
+ }
}
}
}
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/ENodeInitialReducer.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/ENodeInitialReducer.java
new file mode 100644
index 0000000..66cae9b
--- /dev/null
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/ENodeInitialReducer.java
@@ -0,0 +1,59 @@
+package edu.uci.ics.pathmergingh2;
+
+import java.io.IOException;
+import java.util.Iterator;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import edu.uci.ics.genomix.type.KmerUtil;
+
+@SuppressWarnings("deprecation")
+public class ENodeInitialReducer extends MapReduceBase implements
+ Reducer<BytesWritable, MergePathValueWritable, BytesWritable, MergePathValueWritable> {
+ public BytesWritable outputKmer = new BytesWritable();
+ public MergePathValueWritable outputAdjList = new MergePathValueWritable();
+
+
+ @Override
+ public void reduce(BytesWritable key, Iterator<MergePathValueWritable> values,
+ OutputCollector<BytesWritable, MergePathValueWritable> output, Reporter reporter) throws IOException {
+ outputAdjList = values.next();
+ outputKmer.set(key);
+ if (values.hasNext() == true) {
+ byte bitFlag = outputAdjList.getFlag();
+ bitFlag = (byte) (bitFlag & 0xFE);
+ if (bitFlag == 2) {
+ bitFlag = (byte) (0x80 | outputAdjList.getFlag());
+ outputAdjList.set(null, 0, 0, outputAdjList.getAdjBitMap(), bitFlag, outputAdjList.getKmerSize());
+ output.collect(outputKmer, outputAdjList);
+
+ } else {
+ boolean flag = false;
+ while (values.hasNext()) {
+ outputAdjList = values.next();
+ if (outputAdjList.getFlag() == 2) {
+ flag = true;
+ break;
+ }
+ }
+ if (flag == true) {
+ bitFlag = (byte) (0x80 | outputAdjList.getFlag());
+ outputAdjList.set(null, 0, 0, outputAdjList.getAdjBitMap(), bitFlag, outputAdjList.getKmerSize());
+ output.collect(outputKmer, outputAdjList);
+ }
+ }
+ } else {
+ byte bitFlag = outputAdjList.getFlag();
+ bitFlag = (byte) (bitFlag & 0xFE);
+ if (bitFlag == 2) {
+ bitFlag = 0;
+ outputAdjList.set(null, 0, 0, outputAdjList.getAdjBitMap(), bitFlag, outputAdjList.getKmerSize());
+ output.collect(outputKmer, outputAdjList);
+ }
+ }
+ }
+}
+
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Driver.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Driver.java
new file mode 100644
index 0000000..e90e7a4
--- /dev/null
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Driver.java
@@ -0,0 +1,150 @@
+package edu.uci.ics.pathmergingh2;
+
+import java.io.IOException;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hadoop.mapred.SequenceFileOutputFormat;
+import org.apache.hadoop.mapred.TextOutputFormat;
+import org.apache.hadoop.mapred.lib.MultipleOutputs;
+import org.apache.hadoop.mapred.lib.MultipleSequenceFileOutputFormat;
+import org.apache.hadoop.mapred.lib.MultipleTextOutputFormat;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+
+@SuppressWarnings("deprecation")
+public class MergePathH2Driver {
+
+ private static class Options {
+ @Option(name = "-inputpath", usage = "the input path", required = true)
+ public String inputPath;
+
+ @Option(name = "-outputpath", usage = "the output path", required = true)
+ public String outputPath;
+
+ @Option(name = "-mergeresultpath", usage = "the merging results path", required = true)
+ public String mergeResultPath;
+
+ @Option(name = "-num-reducers", usage = "the number of reducers", required = true)
+ public int numReducers;
+
+ @Option(name = "-kmer-size", usage = "the size of kmer", required = true)
+ public int sizeKmer;
+
+ @Option(name = "-merge-rounds", usage = "the while rounds of merging", required = true)
+ public int mergeRound;
+
+ }
+
+
+ public void run(String inputPath, String outputPath, String mergeResultPath, int numReducers, int sizeKmer, int mergeRound, String defaultConfPath)
+ throws IOException{
+
+ JobConf conf = new JobConf(MergePathH2Driver.class);
+ conf.setInt("sizeKmer", sizeKmer);
+
+ if (defaultConfPath != null) {
+ conf.addResource(new Path(defaultConfPath));
+ }
+ conf.setJobName("Initial Path-Starting-Points Table");
+ conf.setMapperClass(SNodeInitialMapper.class);
+ conf.setReducerClass(SNodeInitialReducer.class);
+
+ conf.setMapOutputKeyClass(BytesWritable.class);
+ conf.setMapOutputValueClass(MergePathValueWritable.class);
+
+ conf.setInputFormat(SequenceFileInputFormat.class);
+ conf.setOutputFormat(SequenceFileOutputFormat.class);
+
+ conf.setOutputKeyClass(BytesWritable.class);
+ conf.setOutputValueClass(MergePathValueWritable.class);
+
+ FileInputFormat.setInputPaths(conf, new Path(inputPath));
+ FileOutputFormat.setOutputPath(conf, new Path(inputPath + "-step1"));
+ conf.setNumReduceTasks(numReducers);
+ FileSystem dfs = FileSystem.get(conf);
+ dfs.delete(new Path(inputPath + "-step1"), true);
+ JobClient.runJob(conf);
+/****************************************************************/
+ conf = new JobConf(MergePathH2Driver.class);
+ conf.setInt("sizeKmer", sizeKmer);
+ if (defaultConfPath != null) {
+ conf.addResource(new Path(defaultConfPath));
+ }
+ conf.setJobName("Initial Path-Starting-Points Table");
+ conf.setMapperClass(ENodeInitialMapper.class);
+ conf.setReducerClass(ENodeInitialReducer.class);
+
+ conf.setMapOutputKeyClass(BytesWritable.class);
+ conf.setMapOutputValueClass(MergePathValueWritable.class);
+
+ conf.setInputFormat(SequenceFileInputFormat.class);
+ conf.setOutputFormat(SequenceFileOutputFormat.class);
+
+ conf.setOutputKeyClass(BytesWritable.class);
+ conf.setOutputValueClass(MergePathValueWritable.class);
+
+ FileInputFormat.setInputPaths(conf, new Path(inputPath + "-step1"));
+ FileOutputFormat.setOutputPath(conf, new Path(inputPath + "-step2"));
+ conf.setNumReduceTasks(numReducers);
+ dfs.delete(new Path(inputPath + "-step2"), true);
+ JobClient.runJob(conf);
+// int iMerge = 0;
+/*----------------------------------------------------------------------*/
+/* for(iMerge = 0; iMerge < mergeRound; iMerge ++){
+
+ conf = new JobConf(MergePathDriver.class);
+ conf.setInt("sizeKmer", sizeKmer);
+ conf.setInt("iMerge", iMerge);
+
+ if (defaultConfPath != null) {
+ conf.addResource(new Path(defaultConfPath));
+ }
+ conf.setJobName("Path Merge");
+
+ conf.setMapperClass(MergePathMapper.class);
+ conf.setReducerClass(MergePathReducer.class);
+
+ conf.setMapOutputKeyClass(BytesWritable.class);
+ conf.setMapOutputValueClass(MergePathValueWritable.class);
+
+ conf.setInputFormat(SequenceFileInputFormat.class);
+
+ String uncomplete = "uncomplete" + iMerge;
+ String complete = "complete" + iMerge;
+
+ MultipleOutputs.addNamedOutput(conf, uncomplete,
+ MergePathMultiSeqOutputFormat.class, BytesWritable.class,
+ MergePathValueWritable.class);
+
+ MultipleOutputs.addNamedOutput(conf, complete,
+ MergePathMultiSeqOutputFormat.class, BytesWritable.class,
+ MergePathValueWritable.class);
+
+ conf.setOutputKeyClass(BytesWritable.class);
+ conf.setOutputValueClass(MergePathValueWritable.class);
+
+ FileInputFormat.setInputPaths(conf, new Path(inputPath + "-step1"));
+ FileOutputFormat.setOutputPath(conf, new Path(outputPath));
+ conf.setNumReduceTasks(numReducers);
+ dfs.delete(new Path(outputPath), true);
+ JobClient.runJob(conf);
+ dfs.delete(new Path(inputPath + "-step1"), true);
+ dfs.rename(new Path(outputPath + "/" + uncomplete), new Path(inputPath + "-step1"));
+ dfs.rename(new Path(outputPath + "/" + complete), new Path(mergeResultPath + "/" + complete));
+ }*/
+ }
+
+ public static void main(String[] args) throws Exception {
+ Options options = new Options();
+ CmdLineParser parser = new CmdLineParser(options);
+ parser.parseArgument(args);
+ MergePathH2Driver driver = new MergePathH2Driver();
+ driver.run(options.inputPath, options.outputPath, options.mergeResultPath, options.numReducers, options.sizeKmer, options.mergeRound, null);
+ }
+}
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Mapper.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Mapper.java
new file mode 100644
index 0000000..1d8a98b
--- /dev/null
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Mapper.java
@@ -0,0 +1,83 @@
+package edu.uci.ics.pathmergingh2;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+
+import edu.uci.ics.genomix.type.Kmer;
+import edu.uci.ics.genomix.type.KmerUtil;
+import edu.uci.ics.genomix.type.Kmer.GENE_CODE;
+import edu.uci.ics.pathmerging.MergePathValueWritable;
+
+public class MergePathH2Mapper extends MapReduceBase implements
+ Mapper<BytesWritable, MergePathValueWritable, BytesWritable, MergePathValueWritable> {
+
+ public static int KMER_SIZE;
+ public BytesWritable outputKmer = new BytesWritable();
+ public MergePathValueWritable outputAdjList = new MergePathValueWritable();
+
+ public void configure(JobConf job) {
+ KMER_SIZE = job.getInt("sizeKmer", 0);
+ }
+
+ @Override
+ public void map(BytesWritable key, MergePathValueWritable value,
+ OutputCollector<BytesWritable, MergePathValueWritable> output, Reporter reporter) throws IOException {
+ byte bitFlag = value.getFlag();
+ byte bitStartEnd = (byte) (0x81 & bitFlag);
+ byte succeed = (byte) 0x0F;
+ byte adjBitMap = value.getAdjBitMap();
+ succeed = (byte) (succeed & adjBitMap);
+ byte[] kmerValue = key.getBytes();
+ int kmerLength = key.getLength();
+ switch (bitStartEnd) {
+ case 0x01:
+ byte succeedCode = GENE_CODE.getGeneCodeFromBitMap(succeed);
+ int originalByteNum = Kmer.getByteNumFromK(KMER_SIZE);
+ byte[] tmpKmer = KmerUtil.getLastKmerFromChain(KMER_SIZE, value.getKmerSize(), kmerValue, 0, kmerLength);
+ byte[] newKmer = KmerUtil.shiftKmerWithNextCode(KMER_SIZE, tmpKmer, 0, tmpKmer.length, succeedCode);
+ outputKmer.set(newKmer, 0, originalByteNum);
+
+ int mergeByteNum = Kmer.getByteNumFromK(value.getKmerSize() - (KMER_SIZE - 1));
+ byte[] mergeKmer = KmerUtil.getFirstKmerFromChain(value.getKmerSize() - (KMER_SIZE - 1),
+ value.getKmerSize(), kmerValue, 0, kmerLength);
+
+ bitFlag = (byte) (bitFlag | 0x08);
+ outputAdjList.set(mergeKmer, 0, mergeByteNum, adjBitMap, bitFlag, value.getKmerSize() - (KMER_SIZE - 1));
+ output.collect(outputKmer, outputAdjList);
+ break;
+ case (byte) 0x80:
+ outputKmer.set(key);
+ outputAdjList.set(value);
+ output.collect(outputKmer, outputAdjList);
+ break;
+ case 0x00:
+ succeedCode = GENE_CODE.getGeneCodeFromBitMap(succeed);
+ originalByteNum = Kmer.getByteNumFromK(KMER_SIZE);
+ tmpKmer = KmerUtil.getLastKmerFromChain(KMER_SIZE, value.getKmerSize(), kmerValue, 0, kmerLength);
+ newKmer = KmerUtil.shiftKmerWithNextCode(KMER_SIZE, tmpKmer, 0, tmpKmer.length, succeedCode);
+ outputKmer.set(newKmer, 0, originalByteNum);
+
+ mergeByteNum = Kmer.getByteNumFromK(value.getKmerSize() - (KMER_SIZE - 1));
+ mergeKmer = KmerUtil.getFirstKmerFromChain(value.getKmerSize() - (KMER_SIZE - 1), value.getKmerSize(),
+ kmerValue, 0, kmerLength);
+
+ bitFlag = (byte) (bitFlag | 0x08);
+ outputAdjList
+ .set(mergeKmer, 0, mergeByteNum, adjBitMap, bitFlag, value.getKmerSize() - (KMER_SIZE - 1));
+ output.collect(outputKmer, outputAdjList);
+
+ outputKmer.set(key);
+ outputAdjList.set(value);
+ output.collect(outputKmer, outputAdjList);
+ break;
+ case (byte) 0x81:
+ break;
+ }
+ }
+}
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Reducer.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Reducer.java
new file mode 100644
index 0000000..0551bd3
--- /dev/null
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Reducer.java
@@ -0,0 +1,105 @@
+package edu.uci.ics.pathmergingh2;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.lib.MultipleOutputs;
+
+import edu.uci.ics.genomix.type.Kmer;
+import edu.uci.ics.genomix.type.KmerUtil;
+import edu.uci.ics.pathmerging.MergePathValueWritable;
+
+public class MergePathH2Reducer extends MapReduceBase implements
+ Reducer<BytesWritable, MergePathValueWritable, BytesWritable, MergePathValueWritable> {
+ public BytesWritable outputKmer = new BytesWritable();
+ public static int KMER_SIZE;
+ public MergePathValueWritable outputAdjList = new MergePathValueWritable();
+ MultipleOutputs mos = null;
+ public static int I_MERGE;
+
+ public void configure(JobConf job) {
+ mos = new MultipleOutputs(job);
+ I_MERGE = Integer.parseInt(job.get("iMerge"));
+ KMER_SIZE = job.getInt("sizeKmer", 0);
+ }
+
+ public void reduce(BytesWritable key, Iterator<MergePathValueWritable> values,
+ OutputCollector<BytesWritable, MergePathValueWritable> output, Reporter reporter) throws IOException {
+ outputKmer.set(key);
+ outputAdjList = values.next();
+ if (values.hasNext() == true) {
+ byte[] keyBytes = key.getBytes();
+ int keyLength = key.getLength();
+ byte bitFlag = outputAdjList.getFlag();
+ byte bitStartEnd = (byte) (0x81 & bitFlag);
+ byte bitPosiNegative = (byte) (0x18 & bitFlag);
+ byte succeed = (byte) 0x0F;
+ byte adjBitMap = outputAdjList.getAdjBitMap();
+
+ switch (bitPosiNegative) {
+ case (byte) 0x08:
+ succeed = (byte) (succeed & adjBitMap);
+ if (bitStartEnd == 0x10) {
+ output.collect(outputKmer, outputAdjList);
+ }
+ int kmerSize = outputAdjList.getKmerSize();
+ int mergeByteNum = Kmer.getByteNumFromK(KMER_SIZE + kmerSize);
+ byte[] valueBytes = outputAdjList.getBytes();
+ int valueLength = outputAdjList.getLength();
+ byte[] mergeKmer = KmerUtil.mergeTwoKmer(outputAdjList.getKmerSize(), valueBytes, 0, valueLength,
+ KMER_SIZE, keyBytes, 0, keyLength);
+ outputAdjList = values.next();
+ valueBytes = outputAdjList.getBytes();
+ valueLength = outputAdjList.getLength();
+ adjBitMap = outputAdjList.getAdjBitMap();
+ mergeKmer = KmerUtil.mergeTwoKmer(outputAdjList.getKmerSize(), valueBytes, 0, valueLength,
+ KMER_SIZE + kmerSize, mergeKmer, 0, mergeByteNum);
+ outputKmer.set(mergeKmer, 0, mergeByteNum);
+
+ adjBitMap = (byte) (adjBitMap & 0xF0);
+ adjBitMap = (byte) (adjBitMap | succeed);
+ byte flag = (byte) (bitFlag | 0x80);//zhe you wen ti
+ outputAdjList.set(null, 0, 0, adjBitMap, flag, KMER_SIZE + kmerSize + outputAdjList.getKmerSize());
+ mos.getCollector("uncomplete" + I_MERGE, reporter).collect(outputKmer, outputAdjList);
+ break;
+ case (byte) 0x10:
+ byte[] haha = new byte[outputAdjList.getLength()];
+ for (int i = 0; i < haha.length; i++) {
+ haha[i] = outputAdjList.getBytes()[i];
+ }
+ byte trueadj = outputAdjList.getAdjBitMap();
+ int trueKmersize = outputAdjList.getKmerSize();
+ outputAdjList = values.next();
+ succeed = (byte) (succeed & adjBitMap);
+ if (bitStartEnd == 0x10) {
+ output.collect(outputKmer, outputAdjList);
+ }
+ kmerSize = outputAdjList.getKmerSize();
+ mergeByteNum = Kmer.getByteNumFromK(KMER_SIZE + kmerSize);
+ valueBytes = outputAdjList.getBytes();
+ valueLength = outputAdjList.getLength();
+ mergeKmer = KmerUtil.mergeTwoKmer(outputAdjList.getKmerSize(), valueBytes, 0, valueLength,
+ KMER_SIZE, keyBytes, 0, keyLength);
+ mergeKmer = KmerUtil.mergeTwoKmer(trueKmersize, haha, 0, haha.length,
+ KMER_SIZE + kmerSize, mergeKmer, 0, mergeByteNum);
+ outputKmer.set(mergeKmer, 0, mergeByteNum);
+
+ trueadj = (byte) (adjBitMap & 0xF0);
+ trueadj = (byte) (adjBitMap | succeed);
+// byte flag = (byte) (bitFlag | 0x80);
+ outputAdjList.set(null, 0, 0, trueadj, bitFlag, KMER_SIZE + kmerSize + outputAdjList.getKmerSize());
+ mos.getCollector("uncomplete" + I_MERGE, reporter).collect(outputKmer, outputAdjList);
+ break;
+ }
+ }
+ else{
+ //shi bu shi yijing jieshu ---> complete
+ }
+ }
+}
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathMultiSeqOutputFormat.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathMultiSeqOutputFormat.java
new file mode 100644
index 0000000..cbde512
--- /dev/null
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathMultiSeqOutputFormat.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.pathmergingh2;
+
+import java.io.File;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.mapred.lib.MultipleSequenceFileOutputFormat;
+
+
+public class MergePathMultiSeqOutputFormat extends MultipleSequenceFileOutputFormat<BytesWritable, MergePathValueWritable>{
+ @Override
+ protected String generateLeafFileName(String name) {
+ // TODO Auto-generated method stub System.out.println(name);
+ String[] names = name.split("-");
+ return names[0] + File.separator + name;
+ }
+}
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathMultiTextOutputFormat.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathMultiTextOutputFormat.java
new file mode 100644
index 0000000..d6176e2
--- /dev/null
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathMultiTextOutputFormat.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.pathmergingh2;
+
+import java.io.File;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.lib.MultipleTextOutputFormat;
+
+public class MergePathMultiTextOutputFormat extends MultipleTextOutputFormat<Text, Text>{
+ @Override
+ protected String generateLeafFileName(String name) {
+ // TODO Auto-generated method stub System.out.println(name);
+ String[] names = name.split("-");
+ return names[0] + File.separator + name;
+ }
+}
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathValueWritable.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathValueWritable.java
new file mode 100644
index 0000000..7e944ff
--- /dev/null
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathValueWritable.java
@@ -0,0 +1,151 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.pathmergingh2;
+
+import java.io.IOException;
+import java.io.DataInput;
+import java.io.DataOutput;
+import org.apache.hadoop.io.BinaryComparable;
+import org.apache.hadoop.io.WritableComparable;
+
+import edu.uci.ics.genomix.type.Kmer;
+
+public class MergePathValueWritable extends BinaryComparable implements WritableComparable<BinaryComparable> {
+
+ private static final byte[] EMPTY_BYTES = {};
+ private int size;
+ private byte[] bytes;
+
+ private byte adjBitMap;
+ private byte flag;
+ private int kmerSize;
+
+ public MergePathValueWritable() {
+ this((byte) 0, (byte) 0, (byte) 0, EMPTY_BYTES);
+ }
+
+ public MergePathValueWritable(byte adjBitMap, byte flag, byte kmerSize, byte[] bytes) {
+ this.adjBitMap = adjBitMap;
+ this.flag = flag;
+ this.kmerSize = kmerSize;
+
+ this.bytes = bytes;
+ this.size = bytes.length;
+ }
+
+ public void setSize(int size) {
+ if (size > getCapacity()) {
+ setCapacity(size * 3 / 2);
+ }
+ this.size = size;
+ }
+
+ public int getCapacity() {
+ return bytes.length;
+ }
+
+ public void setCapacity(int new_cap) {
+ if (new_cap != getCapacity()) {
+ byte[] new_data = new byte[new_cap];
+ if (new_cap < size) {
+ size = new_cap;
+ }
+ if (size != 0) {
+ System.arraycopy(bytes, 0, new_data, 0, size);
+ }
+ bytes = new_data;
+ }
+ }
+
+ public void set(MergePathValueWritable newData) {
+ set(newData.bytes, 0, newData.size, newData.adjBitMap, newData.flag, newData.kmerSize);
+ }
+
+ public void set(byte[] newData, int offset, int length, byte adjBitMap, byte flag, int kmerSize) {
+ setSize(0);
+ if (length != 0) {
+ setSize(length);
+ System.arraycopy(newData, offset, bytes, 0, size);
+ }
+ this.adjBitMap = adjBitMap;
+ this.flag = flag;
+ this.kmerSize = kmerSize;
+ }
+
+ @Override
+ public void readFields(DataInput arg0) throws IOException {
+ // TODO Auto-generated method stub
+ setSize(0); // clear the old data
+ setSize(arg0.readInt());
+ if(size != 0){
+ arg0.readFully(bytes, 0, size);
+ }
+ adjBitMap = arg0.readByte();
+ flag = arg0.readByte();
+ kmerSize = arg0.readInt();
+ }
+
+ @Override
+ public void write(DataOutput arg0) throws IOException {
+ // TODO Auto-generated method stub
+ arg0.writeInt(size);
+ arg0.write(bytes, 0, size);
+ arg0.writeByte(adjBitMap);
+ arg0.writeByte(flag);
+ arg0.writeInt(kmerSize);
+ }
+
+ @Override
+ public byte[] getBytes() {
+ // TODO Auto-generated method stub
+ return bytes;
+ }
+
+ @Override
+ public int getLength() {
+ // TODO Auto-generated method stub
+ return size;
+ }
+
+ public byte getAdjBitMap() {
+ return this.adjBitMap;
+ }
+
+ public byte getFlag() {
+ return this.flag;
+ }
+
+ public int getKmerSize() {
+ return this.kmerSize;
+ }
+
+ public String toString() {
+ StringBuffer sb = new StringBuffer(3 * size);
+ for (int idx = 0; idx < size; idx++) {
+ // if not the first, put a blank separator in
+ if (idx != 0) {
+ sb.append(' ');
+ }
+ String num = Integer.toHexString(0xff & bytes[idx]);
+ // if it is only one digit, add a leading 0.
+ if (num.length() < 2) {
+ sb.append('0');
+ }
+ sb.append(num);
+ }
+ return Kmer.GENE_CODE.getSymbolFromBitMap(adjBitMap) + '\t' + String.valueOf(flag) + '\t' + sb.toString();
+ }
+}
+
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/SNodeInitialMapper.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/SNodeInitialMapper.java
new file mode 100644
index 0000000..b58202c
--- /dev/null
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/SNodeInitialMapper.java
@@ -0,0 +1,130 @@
+package edu.uci.ics.pathmergingh2;
+
+import java.io.IOException;
+import org.apache.hadoop.io.ByteWritable;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import edu.uci.ics.genomix.type.KmerUtil;
+import edu.uci.ics.genomix.type.Kmer.GENE_CODE;
+
+@SuppressWarnings("deprecation")
+public class SNodeInitialMapper extends MapReduceBase implements
+ Mapper<BytesWritable, ByteWritable, BytesWritable, MergePathValueWritable> {
+
+ public static int KMER_SIZE;
+ public BytesWritable outputKmer = new BytesWritable();
+ public MergePathValueWritable outputAdjList = new MergePathValueWritable();
+
+ public void configure(JobConf job) {
+ KMER_SIZE = Integer.parseInt(job.get("sizeKmer"));
+ }
+
+ boolean measureDegree(byte adjacent) {
+ boolean result = true;
+ switch (adjacent) {
+ case 0:
+ result = true;
+ break;
+ case 1:
+ result = false;
+ break;
+ case 2:
+ result = false;
+ break;
+ case 3:
+ result = true;
+ break;
+ case 4:
+ result = false;
+ break;
+ case 5:
+ result = true;
+ break;
+ case 6:
+ result = true;
+ break;
+ case 7:
+ result = true;
+ break;
+ case 8:
+ result = false;
+ break;
+ case 9:
+ result = true;
+ break;
+ case 10:
+ result = true;
+ break;
+ case 11:
+ result = true;
+ break;
+ case 12:
+ result = true;
+ break;
+ case 13:
+ result = true;
+ break;
+ case 14:
+ result = true;
+ break;
+ case 15:
+ result = true;
+ break;
+ }
+ return result;
+ }
+
+ @Override
+ public void map(BytesWritable key, ByteWritable value,
+ OutputCollector<BytesWritable, MergePathValueWritable> output, Reporter reporter) throws IOException {
+ byte precursor = (byte) 0xF0;
+ byte succeed = (byte) 0x0F;
+ byte adjBitMap = value.get();
+ byte bitFlag = (byte) 0;
+ precursor = (byte) (precursor & adjBitMap);
+ precursor = (byte) ((precursor & 0xff) >> 4);
+ succeed = (byte) (succeed & adjBitMap);
+ boolean inDegree = measureDegree(precursor);
+ boolean outDegree = measureDegree(succeed);
+ byte[] kmerValue = key.getBytes();
+ int kmerLength = key.getLength();
+ if (inDegree == false && outDegree == false) {//bitflag = 2, it means that it has no need to store the precurCode.
+ outputKmer.set(key);
+ bitFlag = (byte) 2;
+ outputAdjList.set(null, 0, 0, adjBitMap, bitFlag, KMER_SIZE);
+ output.collect(outputKmer, outputAdjList);
+ }
+ else{
+ for(int i = 0 ; i < 4; i ++){
+ byte temp = 0x01;
+ temp = (byte)(temp << i);
+ temp = (byte) (succeed & temp);
+ if(temp != 0 ){
+ byte succeedCode = GENE_CODE.getGeneCodeFromBitMap(temp);
+ byte[] newKmer = KmerUtil.shiftKmerWithNextCode(KMER_SIZE, kmerValue, 0, kmerLength, succeedCode);
+ outputKmer.set(newKmer, 0, kmerLength);
+ bitFlag = (byte) 0x80;
+ outputAdjList.set(null, 0, 0, (byte)0, bitFlag, KMER_SIZE);
+ output.collect(outputKmer, outputAdjList);
+ }
+ }
+ for(int i = 0; i < 4;i ++){
+ byte temp = 0x01;
+ temp = (byte)(temp << i);
+ temp = (byte)(precursor & temp);
+ if(temp != 0){
+ byte precurCode = GENE_CODE.getGeneCodeFromBitMap(temp);
+ byte[] newKmer = KmerUtil.shiftKmerWithPreCode(KMER_SIZE, kmerValue, 0, kmerLength, precurCode);
+ outputKmer.set(newKmer, 0, kmerLength);
+ bitFlag = (byte)0x40;
+ outputAdjList.set(null, 0, 0, (byte)0, bitFlag, KMER_SIZE);
+ output.collect(outputKmer, outputAdjList);
+ }
+ }
+ }
+ }
+}
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/SNodeInitialReducer.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/SNodeInitialReducer.java
new file mode 100644
index 0000000..7b918b6
--- /dev/null
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/SNodeInitialReducer.java
@@ -0,0 +1,72 @@
+package edu.uci.ics.pathmergingh2;
+
+import java.io.IOException;
+import java.util.Iterator;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+
+import edu.uci.ics.genomix.type.KmerUtil;
+
+@SuppressWarnings("deprecation")
+public class SNodeInitialReducer extends MapReduceBase implements
+ Reducer<BytesWritable, MergePathValueWritable, BytesWritable, MergePathValueWritable> {
+ public BytesWritable outputKmer = new BytesWritable();
+ public MergePathValueWritable outputAdjList = new MergePathValueWritable();
+ public static int KMER_SIZE;
+
+ public void configure(JobConf job) {
+ KMER_SIZE = Integer.parseInt(job.get("sizeKmer"));
+ }
+ @Override
+ public void reduce(BytesWritable key, Iterator<MergePathValueWritable> values,
+ OutputCollector<BytesWritable, MergePathValueWritable> output, Reporter reporter) throws IOException {
+ outputAdjList = values.next();
+ outputKmer.set(key);
+ byte[] kmerValue = key.getBytes();
+ int kmerLength = key.getLength();
+ if (values.hasNext() == true) {
+ if (outputAdjList.getFlag() == 2) {
+ byte bitFlag = 1;
+ outputAdjList.set(null, 0, 0, outputAdjList.getAdjBitMap(), bitFlag, outputAdjList.getKmerSize());
+ output.collect(outputKmer, outputAdjList);
+ } else {
+ boolean flag = false;
+ while (values.hasNext()) {
+ outputAdjList = values.next();
+ if (outputAdjList.getFlag() == 2) {
+ flag = true;
+ break;
+ }
+ else{
+
+ }
+ }
+ if (flag == true) {
+ byte bitFlag = 1;
+ outputAdjList.set(null, 0, 0, outputAdjList.getAdjBitMap(), bitFlag, outputAdjList.getKmerSize());
+ output.collect(outputKmer, outputAdjList);
+ }
+ }
+ } else {
+ if (outputAdjList.getFlag() == 2) {
+ byte bitFlag = 0;
+ outputAdjList.set(null, 0, 0, outputAdjList.getAdjBitMap(), bitFlag, outputAdjList.getKmerSize());
+ output.collect(outputKmer, outputAdjList);
+ }
+ else {
+ byte bitFlag = outputAdjList.getFlag();
+ byte adjBitMap = outputAdjList.getAdjBitMap();
+ byte precurCode = (byte) ((0xC0 & bitFlag) >> 6);
+ bitFlag = 0;
+ byte[] newKmer = KmerUtil.shiftKmerWithPreCode(KMER_SIZE, kmerValue, 0, kmerLength, precurCode);
+ outputKmer.set(newKmer, 0, kmerLength);
+ outputAdjList.set(null, 0, 0, adjBitMap, bitFlag, kmerLength);
+ output.collect(outputKmer, outputAdjList);
+ }
+ }
+ }
+}
diff --git a/genomix/genomix-hadoop/src/test/java/edu/uci/ics/pathmergingh2/MergePathH2Test.java b/genomix/genomix-hadoop/src/test/java/edu/uci/ics/pathmergingh2/MergePathH2Test.java
new file mode 100644
index 0000000..3e66214
--- /dev/null
+++ b/genomix/genomix-hadoop/src/test/java/edu/uci/ics/pathmergingh2/MergePathH2Test.java
@@ -0,0 +1,103 @@
+package edu.uci.ics.pathmergingh2;
+
+import java.io.BufferedWriter;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.junit.Test;
+import edu.uci.ics.utils.TestUtils;
+
+@SuppressWarnings("deprecation")
+public class MergePathH2Test {
+ private static final String ACTUAL_RESULT_DIR = "actual3";
+ private static final String COMPARE_DIR = "compare";
+ private JobConf conf = new JobConf();
+ private static final String HADOOP_CONF_PATH = ACTUAL_RESULT_DIR + File.separator + "conf.xml";
+ private static final String DATA_PATH = "actual2" + "/result2" + "/part-00000";
+ private static final String HDFS_PATH = "/webmap";
+ private static final String HDFA_PATH_DATA = "/webmapdata";
+
+ private static final String RESULT_PATH = "/result3";
+ private static final String EXPECTED_PATH = "expected/result3";
+ private static final String TEST_SOURCE_DIR = COMPARE_DIR + RESULT_PATH + "/comparesource.txt";
+ private static final int COUNT_REDUCER = 4;
+ private static final int SIZE_KMER = 3;
+
+ private MiniDFSCluster dfsCluster;
+ private MiniMRCluster mrCluster;
+ private FileSystem dfs;
+
+ @SuppressWarnings("resource")
+ @Test
+ public void test() throws Exception {
+ FileUtils.forceMkdir(new File(ACTUAL_RESULT_DIR));
+ FileUtils.cleanDirectory(new File(ACTUAL_RESULT_DIR));
+ startHadoop();
+
+ MergePathH2Driver tldriver = new MergePathH2Driver();
+
+ tldriver.run(HDFS_PATH, RESULT_PATH, HDFA_PATH_DATA, COUNT_REDUCER, SIZE_KMER, 3, HADOOP_CONF_PATH);
+
+/* SequenceFile.Reader reader = null;
+ Path path = new Path(HDFA_PATH_DATA + "/complete2" + "/complete2-r-00000");
+ reader = new SequenceFile.Reader(dfs, path, conf);
+ BytesWritable key = (BytesWritable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
+ MergePathValueWritable value = (MergePathValueWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
+ File filePathTo = new File(TEST_SOURCE_DIR);
+ BufferedWriter bw = new BufferedWriter(new FileWriter(filePathTo));
+ while (reader.next(key, value)) {
+ bw.write(key.toString() + "\t" + value.getAdjBitMap() + "\t" + value.getFlag());
+ bw.newLine();
+ }
+ bw.close();
+
+ dumpResult();
+ TestUtils.compareWithResult(new File(TEST_SOURCE_DIR), new File(EXPECTED_PATH));
+*/
+ cleanupHadoop();
+
+ }
+ private void startHadoop() throws IOException {
+ FileSystem lfs = FileSystem.getLocal(new Configuration());
+ lfs.delete(new Path("build"), true);
+ System.setProperty("hadoop.log.dir", "logs");
+ dfsCluster = new MiniDFSCluster(conf, 2, true, null);
+ dfs = dfsCluster.getFileSystem();
+ mrCluster = new MiniMRCluster(4, dfs.getUri().toString(), 2);
+
+ Path src = new Path(DATA_PATH);
+ Path dest = new Path(HDFS_PATH + "/");
+ dfs.mkdirs(dest);
+ dfs.copyFromLocalFile(src, dest);
+ Path data = new Path(HDFA_PATH_DATA + "/");
+ dfs.mkdirs(data);
+
+ DataOutputStream confOutput = new DataOutputStream(new FileOutputStream(new File(HADOOP_CONF_PATH)));
+ conf.writeXml(confOutput);
+ confOutput.flush();
+ confOutput.close();
+ }
+
+ private void cleanupHadoop() throws IOException {
+ mrCluster.shutdown();
+ dfsCluster.shutdown();
+ }
+
+ private void dumpResult() throws IOException {
+ Path src = new Path(HDFA_PATH_DATA + "/" + "complete2");
+ Path dest = new Path(ACTUAL_RESULT_DIR + "/");
+ dfs.copyToLocalFile(src, dest);
+ }
+}