Please enter the commit message for your changes. Lines starting
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/ENodeInitialReducer.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/ENodeInitialReducer.java
deleted file mode 100644
index 66cae9b..0000000
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/ENodeInitialReducer.java
+++ /dev/null
@@ -1,59 +0,0 @@
-package edu.uci.ics.pathmergingh2;
-
-import java.io.IOException;
-import java.util.Iterator;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-import edu.uci.ics.genomix.type.KmerUtil;
-
-@SuppressWarnings("deprecation")
-public class ENodeInitialReducer extends MapReduceBase implements
- Reducer<BytesWritable, MergePathValueWritable, BytesWritable, MergePathValueWritable> {
- public BytesWritable outputKmer = new BytesWritable();
- public MergePathValueWritable outputAdjList = new MergePathValueWritable();
-
-
- @Override
- public void reduce(BytesWritable key, Iterator<MergePathValueWritable> values,
- OutputCollector<BytesWritable, MergePathValueWritable> output, Reporter reporter) throws IOException {
- outputAdjList = values.next();
- outputKmer.set(key);
- if (values.hasNext() == true) {
- byte bitFlag = outputAdjList.getFlag();
- bitFlag = (byte) (bitFlag & 0xFE);
- if (bitFlag == 2) {
- bitFlag = (byte) (0x80 | outputAdjList.getFlag());
- outputAdjList.set(null, 0, 0, outputAdjList.getAdjBitMap(), bitFlag, outputAdjList.getKmerSize());
- output.collect(outputKmer, outputAdjList);
-
- } else {
- boolean flag = false;
- while (values.hasNext()) {
- outputAdjList = values.next();
- if (outputAdjList.getFlag() == 2) {
- flag = true;
- break;
- }
- }
- if (flag == true) {
- bitFlag = (byte) (0x80 | outputAdjList.getFlag());
- outputAdjList.set(null, 0, 0, outputAdjList.getAdjBitMap(), bitFlag, outputAdjList.getKmerSize());
- output.collect(outputKmer, outputAdjList);
- }
- }
- } else {
- byte bitFlag = outputAdjList.getFlag();
- bitFlag = (byte) (bitFlag & 0xFE);
- if (bitFlag == 2) {
- bitFlag = 0;
- outputAdjList.set(null, 0, 0, outputAdjList.getAdjBitMap(), bitFlag, outputAdjList.getKmerSize());
- output.collect(outputKmer, outputAdjList);
- }
- }
- }
-}
-
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Driver.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Driver.java
deleted file mode 100644
index e90e7a4..0000000
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Driver.java
+++ /dev/null
@@ -1,150 +0,0 @@
-package edu.uci.ics.pathmergingh2;
-
-import java.io.IOException;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.SequenceFileInputFormat;
-import org.apache.hadoop.mapred.SequenceFileOutputFormat;
-import org.apache.hadoop.mapred.TextOutputFormat;
-import org.apache.hadoop.mapred.lib.MultipleOutputs;
-import org.apache.hadoop.mapred.lib.MultipleSequenceFileOutputFormat;
-import org.apache.hadoop.mapred.lib.MultipleTextOutputFormat;
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
-@SuppressWarnings("deprecation")
-public class MergePathH2Driver {
-
- private static class Options {
- @Option(name = "-inputpath", usage = "the input path", required = true)
- public String inputPath;
-
- @Option(name = "-outputpath", usage = "the output path", required = true)
- public String outputPath;
-
- @Option(name = "-mergeresultpath", usage = "the merging results path", required = true)
- public String mergeResultPath;
-
- @Option(name = "-num-reducers", usage = "the number of reducers", required = true)
- public int numReducers;
-
- @Option(name = "-kmer-size", usage = "the size of kmer", required = true)
- public int sizeKmer;
-
- @Option(name = "-merge-rounds", usage = "the while rounds of merging", required = true)
- public int mergeRound;
-
- }
-
-
- public void run(String inputPath, String outputPath, String mergeResultPath, int numReducers, int sizeKmer, int mergeRound, String defaultConfPath)
- throws IOException{
-
- JobConf conf = new JobConf(MergePathH2Driver.class);
- conf.setInt("sizeKmer", sizeKmer);
-
- if (defaultConfPath != null) {
- conf.addResource(new Path(defaultConfPath));
- }
- conf.setJobName("Initial Path-Starting-Points Table");
- conf.setMapperClass(SNodeInitialMapper.class);
- conf.setReducerClass(SNodeInitialReducer.class);
-
- conf.setMapOutputKeyClass(BytesWritable.class);
- conf.setMapOutputValueClass(MergePathValueWritable.class);
-
- conf.setInputFormat(SequenceFileInputFormat.class);
- conf.setOutputFormat(SequenceFileOutputFormat.class);
-
- conf.setOutputKeyClass(BytesWritable.class);
- conf.setOutputValueClass(MergePathValueWritable.class);
-
- FileInputFormat.setInputPaths(conf, new Path(inputPath));
- FileOutputFormat.setOutputPath(conf, new Path(inputPath + "-step1"));
- conf.setNumReduceTasks(numReducers);
- FileSystem dfs = FileSystem.get(conf);
- dfs.delete(new Path(inputPath + "-step1"), true);
- JobClient.runJob(conf);
-/****************************************************************/
- conf = new JobConf(MergePathH2Driver.class);
- conf.setInt("sizeKmer", sizeKmer);
- if (defaultConfPath != null) {
- conf.addResource(new Path(defaultConfPath));
- }
- conf.setJobName("Initial Path-Starting-Points Table");
- conf.setMapperClass(ENodeInitialMapper.class);
- conf.setReducerClass(ENodeInitialReducer.class);
-
- conf.setMapOutputKeyClass(BytesWritable.class);
- conf.setMapOutputValueClass(MergePathValueWritable.class);
-
- conf.setInputFormat(SequenceFileInputFormat.class);
- conf.setOutputFormat(SequenceFileOutputFormat.class);
-
- conf.setOutputKeyClass(BytesWritable.class);
- conf.setOutputValueClass(MergePathValueWritable.class);
-
- FileInputFormat.setInputPaths(conf, new Path(inputPath + "-step1"));
- FileOutputFormat.setOutputPath(conf, new Path(inputPath + "-step2"));
- conf.setNumReduceTasks(numReducers);
- dfs.delete(new Path(inputPath + "-step2"), true);
- JobClient.runJob(conf);
-// int iMerge = 0;
-/*----------------------------------------------------------------------*/
-/* for(iMerge = 0; iMerge < mergeRound; iMerge ++){
-
- conf = new JobConf(MergePathDriver.class);
- conf.setInt("sizeKmer", sizeKmer);
- conf.setInt("iMerge", iMerge);
-
- if (defaultConfPath != null) {
- conf.addResource(new Path(defaultConfPath));
- }
- conf.setJobName("Path Merge");
-
- conf.setMapperClass(MergePathMapper.class);
- conf.setReducerClass(MergePathReducer.class);
-
- conf.setMapOutputKeyClass(BytesWritable.class);
- conf.setMapOutputValueClass(MergePathValueWritable.class);
-
- conf.setInputFormat(SequenceFileInputFormat.class);
-
- String uncomplete = "uncomplete" + iMerge;
- String complete = "complete" + iMerge;
-
- MultipleOutputs.addNamedOutput(conf, uncomplete,
- MergePathMultiSeqOutputFormat.class, BytesWritable.class,
- MergePathValueWritable.class);
-
- MultipleOutputs.addNamedOutput(conf, complete,
- MergePathMultiSeqOutputFormat.class, BytesWritable.class,
- MergePathValueWritable.class);
-
- conf.setOutputKeyClass(BytesWritable.class);
- conf.setOutputValueClass(MergePathValueWritable.class);
-
- FileInputFormat.setInputPaths(conf, new Path(inputPath + "-step1"));
- FileOutputFormat.setOutputPath(conf, new Path(outputPath));
- conf.setNumReduceTasks(numReducers);
- dfs.delete(new Path(outputPath), true);
- JobClient.runJob(conf);
- dfs.delete(new Path(inputPath + "-step1"), true);
- dfs.rename(new Path(outputPath + "/" + uncomplete), new Path(inputPath + "-step1"));
- dfs.rename(new Path(outputPath + "/" + complete), new Path(mergeResultPath + "/" + complete));
- }*/
- }
-
- public static void main(String[] args) throws Exception {
- Options options = new Options();
- CmdLineParser parser = new CmdLineParser(options);
- parser.parseArgument(args);
- MergePathH2Driver driver = new MergePathH2Driver();
- driver.run(options.inputPath, options.outputPath, options.mergeResultPath, options.numReducers, options.sizeKmer, options.mergeRound, null);
- }
-}
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Mapper.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Mapper.java
deleted file mode 100644
index 1d8a98b..0000000
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Mapper.java
+++ /dev/null
@@ -1,83 +0,0 @@
-package edu.uci.ics.pathmergingh2;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reporter;
-
-import edu.uci.ics.genomix.type.Kmer;
-import edu.uci.ics.genomix.type.KmerUtil;
-import edu.uci.ics.genomix.type.Kmer.GENE_CODE;
-import edu.uci.ics.pathmerging.MergePathValueWritable;
-
-public class MergePathH2Mapper extends MapReduceBase implements
- Mapper<BytesWritable, MergePathValueWritable, BytesWritable, MergePathValueWritable> {
-
- public static int KMER_SIZE;
- public BytesWritable outputKmer = new BytesWritable();
- public MergePathValueWritable outputAdjList = new MergePathValueWritable();
-
- public void configure(JobConf job) {
- KMER_SIZE = job.getInt("sizeKmer", 0);
- }
-
- @Override
- public void map(BytesWritable key, MergePathValueWritable value,
- OutputCollector<BytesWritable, MergePathValueWritable> output, Reporter reporter) throws IOException {
- byte bitFlag = value.getFlag();
- byte bitStartEnd = (byte) (0x81 & bitFlag);
- byte succeed = (byte) 0x0F;
- byte adjBitMap = value.getAdjBitMap();
- succeed = (byte) (succeed & adjBitMap);
- byte[] kmerValue = key.getBytes();
- int kmerLength = key.getLength();
- switch (bitStartEnd) {
- case 0x01:
- byte succeedCode = GENE_CODE.getGeneCodeFromBitMap(succeed);
- int originalByteNum = Kmer.getByteNumFromK(KMER_SIZE);
- byte[] tmpKmer = KmerUtil.getLastKmerFromChain(KMER_SIZE, value.getKmerSize(), kmerValue, 0, kmerLength);
- byte[] newKmer = KmerUtil.shiftKmerWithNextCode(KMER_SIZE, tmpKmer, 0, tmpKmer.length, succeedCode);
- outputKmer.set(newKmer, 0, originalByteNum);
-
- int mergeByteNum = Kmer.getByteNumFromK(value.getKmerSize() - (KMER_SIZE - 1));
- byte[] mergeKmer = KmerUtil.getFirstKmerFromChain(value.getKmerSize() - (KMER_SIZE - 1),
- value.getKmerSize(), kmerValue, 0, kmerLength);
-
- bitFlag = (byte) (bitFlag | 0x08);
- outputAdjList.set(mergeKmer, 0, mergeByteNum, adjBitMap, bitFlag, value.getKmerSize() - (KMER_SIZE - 1));
- output.collect(outputKmer, outputAdjList);
- break;
- case (byte) 0x80:
- outputKmer.set(key);
- outputAdjList.set(value);
- output.collect(outputKmer, outputAdjList);
- break;
- case 0x00:
- succeedCode = GENE_CODE.getGeneCodeFromBitMap(succeed);
- originalByteNum = Kmer.getByteNumFromK(KMER_SIZE);
- tmpKmer = KmerUtil.getLastKmerFromChain(KMER_SIZE, value.getKmerSize(), kmerValue, 0, kmerLength);
- newKmer = KmerUtil.shiftKmerWithNextCode(KMER_SIZE, tmpKmer, 0, tmpKmer.length, succeedCode);
- outputKmer.set(newKmer, 0, originalByteNum);
-
- mergeByteNum = Kmer.getByteNumFromK(value.getKmerSize() - (KMER_SIZE - 1));
- mergeKmer = KmerUtil.getFirstKmerFromChain(value.getKmerSize() - (KMER_SIZE - 1), value.getKmerSize(),
- kmerValue, 0, kmerLength);
-
- bitFlag = (byte) (bitFlag | 0x08);
- outputAdjList
- .set(mergeKmer, 0, mergeByteNum, adjBitMap, bitFlag, value.getKmerSize() - (KMER_SIZE - 1));
- output.collect(outputKmer, outputAdjList);
-
- outputKmer.set(key);
- outputAdjList.set(value);
- output.collect(outputKmer, outputAdjList);
- break;
- case (byte) 0x81:
- break;
- }
- }
-}
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Reducer.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Reducer.java
deleted file mode 100644
index 0551bd3..0000000
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathH2Reducer.java
+++ /dev/null
@@ -1,105 +0,0 @@
-package edu.uci.ics.pathmergingh2;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.lib.MultipleOutputs;
-
-import edu.uci.ics.genomix.type.Kmer;
-import edu.uci.ics.genomix.type.KmerUtil;
-import edu.uci.ics.pathmerging.MergePathValueWritable;
-
-public class MergePathH2Reducer extends MapReduceBase implements
- Reducer<BytesWritable, MergePathValueWritable, BytesWritable, MergePathValueWritable> {
- public BytesWritable outputKmer = new BytesWritable();
- public static int KMER_SIZE;
- public MergePathValueWritable outputAdjList = new MergePathValueWritable();
- MultipleOutputs mos = null;
- public static int I_MERGE;
-
- public void configure(JobConf job) {
- mos = new MultipleOutputs(job);
- I_MERGE = Integer.parseInt(job.get("iMerge"));
- KMER_SIZE = job.getInt("sizeKmer", 0);
- }
-
- public void reduce(BytesWritable key, Iterator<MergePathValueWritable> values,
- OutputCollector<BytesWritable, MergePathValueWritable> output, Reporter reporter) throws IOException {
- outputKmer.set(key);
- outputAdjList = values.next();
- if (values.hasNext() == true) {
- byte[] keyBytes = key.getBytes();
- int keyLength = key.getLength();
- byte bitFlag = outputAdjList.getFlag();
- byte bitStartEnd = (byte) (0x81 & bitFlag);
- byte bitPosiNegative = (byte) (0x18 & bitFlag);
- byte succeed = (byte) 0x0F;
- byte adjBitMap = outputAdjList.getAdjBitMap();
-
- switch (bitPosiNegative) {
- case (byte) 0x08:
- succeed = (byte) (succeed & adjBitMap);
- if (bitStartEnd == 0x10) {
- output.collect(outputKmer, outputAdjList);
- }
- int kmerSize = outputAdjList.getKmerSize();
- int mergeByteNum = Kmer.getByteNumFromK(KMER_SIZE + kmerSize);
- byte[] valueBytes = outputAdjList.getBytes();
- int valueLength = outputAdjList.getLength();
- byte[] mergeKmer = KmerUtil.mergeTwoKmer(outputAdjList.getKmerSize(), valueBytes, 0, valueLength,
- KMER_SIZE, keyBytes, 0, keyLength);
- outputAdjList = values.next();
- valueBytes = outputAdjList.getBytes();
- valueLength = outputAdjList.getLength();
- adjBitMap = outputAdjList.getAdjBitMap();
- mergeKmer = KmerUtil.mergeTwoKmer(outputAdjList.getKmerSize(), valueBytes, 0, valueLength,
- KMER_SIZE + kmerSize, mergeKmer, 0, mergeByteNum);
- outputKmer.set(mergeKmer, 0, mergeByteNum);
-
- adjBitMap = (byte) (adjBitMap & 0xF0);
- adjBitMap = (byte) (adjBitMap | succeed);
- byte flag = (byte) (bitFlag | 0x80);//zhe you wen ti
- outputAdjList.set(null, 0, 0, adjBitMap, flag, KMER_SIZE + kmerSize + outputAdjList.getKmerSize());
- mos.getCollector("uncomplete" + I_MERGE, reporter).collect(outputKmer, outputAdjList);
- break;
- case (byte) 0x10:
- byte[] haha = new byte[outputAdjList.getLength()];
- for (int i = 0; i < haha.length; i++) {
- haha[i] = outputAdjList.getBytes()[i];
- }
- byte trueadj = outputAdjList.getAdjBitMap();
- int trueKmersize = outputAdjList.getKmerSize();
- outputAdjList = values.next();
- succeed = (byte) (succeed & adjBitMap);
- if (bitStartEnd == 0x10) {
- output.collect(outputKmer, outputAdjList);
- }
- kmerSize = outputAdjList.getKmerSize();
- mergeByteNum = Kmer.getByteNumFromK(KMER_SIZE + kmerSize);
- valueBytes = outputAdjList.getBytes();
- valueLength = outputAdjList.getLength();
- mergeKmer = KmerUtil.mergeTwoKmer(outputAdjList.getKmerSize(), valueBytes, 0, valueLength,
- KMER_SIZE, keyBytes, 0, keyLength);
- mergeKmer = KmerUtil.mergeTwoKmer(trueKmersize, haha, 0, haha.length,
- KMER_SIZE + kmerSize, mergeKmer, 0, mergeByteNum);
- outputKmer.set(mergeKmer, 0, mergeByteNum);
-
- trueadj = (byte) (adjBitMap & 0xF0);
- trueadj = (byte) (adjBitMap | succeed);
-// byte flag = (byte) (bitFlag | 0x80);
- outputAdjList.set(null, 0, 0, trueadj, bitFlag, KMER_SIZE + kmerSize + outputAdjList.getKmerSize());
- mos.getCollector("uncomplete" + I_MERGE, reporter).collect(outputKmer, outputAdjList);
- break;
- }
- }
- else{
- //shi bu shi yijing jieshu ---> complete
- }
- }
-}
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathMultiSeqOutputFormat.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathMultiSeqOutputFormat.java
deleted file mode 100644
index cbde512..0000000
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathMultiSeqOutputFormat.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright 2009-2012 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.pathmergingh2;
-
-import java.io.File;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.mapred.lib.MultipleSequenceFileOutputFormat;
-
-
-public class MergePathMultiSeqOutputFormat extends MultipleSequenceFileOutputFormat<BytesWritable, MergePathValueWritable>{
- @Override
- protected String generateLeafFileName(String name) {
- // TODO Auto-generated method stub System.out.println(name);
- String[] names = name.split("-");
- return names[0] + File.separator + name;
- }
-}
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathMultiTextOutputFormat.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathMultiTextOutputFormat.java
deleted file mode 100644
index d6176e2..0000000
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathMultiTextOutputFormat.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright 2009-2012 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.pathmergingh2;
-
-import java.io.File;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.lib.MultipleTextOutputFormat;
-
-public class MergePathMultiTextOutputFormat extends MultipleTextOutputFormat<Text, Text>{
- @Override
- protected String generateLeafFileName(String name) {
- // TODO Auto-generated method stub System.out.println(name);
- String[] names = name.split("-");
- return names[0] + File.separator + name;
- }
-}
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathValueWritable.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathValueWritable.java
deleted file mode 100644
index 7e944ff..0000000
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/MergePathValueWritable.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Copyright 2009-2012 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.pathmergingh2;
-
-import java.io.IOException;
-import java.io.DataInput;
-import java.io.DataOutput;
-import org.apache.hadoop.io.BinaryComparable;
-import org.apache.hadoop.io.WritableComparable;
-
-import edu.uci.ics.genomix.type.Kmer;
-
-public class MergePathValueWritable extends BinaryComparable implements WritableComparable<BinaryComparable> {
-
- private static final byte[] EMPTY_BYTES = {};
- private int size;
- private byte[] bytes;
-
- private byte adjBitMap;
- private byte flag;
- private int kmerSize;
-
- public MergePathValueWritable() {
- this((byte) 0, (byte) 0, (byte) 0, EMPTY_BYTES);
- }
-
- public MergePathValueWritable(byte adjBitMap, byte flag, byte kmerSize, byte[] bytes) {
- this.adjBitMap = adjBitMap;
- this.flag = flag;
- this.kmerSize = kmerSize;
-
- this.bytes = bytes;
- this.size = bytes.length;
- }
-
- public void setSize(int size) {
- if (size > getCapacity()) {
- setCapacity(size * 3 / 2);
- }
- this.size = size;
- }
-
- public int getCapacity() {
- return bytes.length;
- }
-
- public void setCapacity(int new_cap) {
- if (new_cap != getCapacity()) {
- byte[] new_data = new byte[new_cap];
- if (new_cap < size) {
- size = new_cap;
- }
- if (size != 0) {
- System.arraycopy(bytes, 0, new_data, 0, size);
- }
- bytes = new_data;
- }
- }
-
- public void set(MergePathValueWritable newData) {
- set(newData.bytes, 0, newData.size, newData.adjBitMap, newData.flag, newData.kmerSize);
- }
-
- public void set(byte[] newData, int offset, int length, byte adjBitMap, byte flag, int kmerSize) {
- setSize(0);
- if (length != 0) {
- setSize(length);
- System.arraycopy(newData, offset, bytes, 0, size);
- }
- this.adjBitMap = adjBitMap;
- this.flag = flag;
- this.kmerSize = kmerSize;
- }
-
- @Override
- public void readFields(DataInput arg0) throws IOException {
- // TODO Auto-generated method stub
- setSize(0); // clear the old data
- setSize(arg0.readInt());
- if(size != 0){
- arg0.readFully(bytes, 0, size);
- }
- adjBitMap = arg0.readByte();
- flag = arg0.readByte();
- kmerSize = arg0.readInt();
- }
-
- @Override
- public void write(DataOutput arg0) throws IOException {
- // TODO Auto-generated method stub
- arg0.writeInt(size);
- arg0.write(bytes, 0, size);
- arg0.writeByte(adjBitMap);
- arg0.writeByte(flag);
- arg0.writeInt(kmerSize);
- }
-
- @Override
- public byte[] getBytes() {
- // TODO Auto-generated method stub
- return bytes;
- }
-
- @Override
- public int getLength() {
- // TODO Auto-generated method stub
- return size;
- }
-
- public byte getAdjBitMap() {
- return this.adjBitMap;
- }
-
- public byte getFlag() {
- return this.flag;
- }
-
- public int getKmerSize() {
- return this.kmerSize;
- }
-
- public String toString() {
- StringBuffer sb = new StringBuffer(3 * size);
- for (int idx = 0; idx < size; idx++) {
- // if not the first, put a blank separator in
- if (idx != 0) {
- sb.append(' ');
- }
- String num = Integer.toHexString(0xff & bytes[idx]);
- // if it is only one digit, add a leading 0.
- if (num.length() < 2) {
- sb.append('0');
- }
- sb.append(num);
- }
- return Kmer.GENE_CODE.getSymbolFromBitMap(adjBitMap) + '\t' + String.valueOf(flag) + '\t' + sb.toString();
- }
-}
-
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/SNodeInitialMapper.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/SNodeInitialMapper.java
deleted file mode 100644
index b58202c..0000000
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/SNodeInitialMapper.java
+++ /dev/null
@@ -1,130 +0,0 @@
-package edu.uci.ics.pathmergingh2;
-
-import java.io.IOException;
-import org.apache.hadoop.io.ByteWritable;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reporter;
-import edu.uci.ics.genomix.type.KmerUtil;
-import edu.uci.ics.genomix.type.Kmer.GENE_CODE;
-
-@SuppressWarnings("deprecation")
-public class SNodeInitialMapper extends MapReduceBase implements
- Mapper<BytesWritable, ByteWritable, BytesWritable, MergePathValueWritable> {
-
- public static int KMER_SIZE;
- public BytesWritable outputKmer = new BytesWritable();
- public MergePathValueWritable outputAdjList = new MergePathValueWritable();
-
- public void configure(JobConf job) {
- KMER_SIZE = Integer.parseInt(job.get("sizeKmer"));
- }
-
- boolean measureDegree(byte adjacent) {
- boolean result = true;
- switch (adjacent) {
- case 0:
- result = true;
- break;
- case 1:
- result = false;
- break;
- case 2:
- result = false;
- break;
- case 3:
- result = true;
- break;
- case 4:
- result = false;
- break;
- case 5:
- result = true;
- break;
- case 6:
- result = true;
- break;
- case 7:
- result = true;
- break;
- case 8:
- result = false;
- break;
- case 9:
- result = true;
- break;
- case 10:
- result = true;
- break;
- case 11:
- result = true;
- break;
- case 12:
- result = true;
- break;
- case 13:
- result = true;
- break;
- case 14:
- result = true;
- break;
- case 15:
- result = true;
- break;
- }
- return result;
- }
-
- @Override
- public void map(BytesWritable key, ByteWritable value,
- OutputCollector<BytesWritable, MergePathValueWritable> output, Reporter reporter) throws IOException {
- byte precursor = (byte) 0xF0;
- byte succeed = (byte) 0x0F;
- byte adjBitMap = value.get();
- byte bitFlag = (byte) 0;
- precursor = (byte) (precursor & adjBitMap);
- precursor = (byte) ((precursor & 0xff) >> 4);
- succeed = (byte) (succeed & adjBitMap);
- boolean inDegree = measureDegree(precursor);
- boolean outDegree = measureDegree(succeed);
- byte[] kmerValue = key.getBytes();
- int kmerLength = key.getLength();
- if (inDegree == false && outDegree == false) {//bitflag = 2, it means that it has no need to store the precurCode.
- outputKmer.set(key);
- bitFlag = (byte) 2;
- outputAdjList.set(null, 0, 0, adjBitMap, bitFlag, KMER_SIZE);
- output.collect(outputKmer, outputAdjList);
- }
- else{
- for(int i = 0 ; i < 4; i ++){
- byte temp = 0x01;
- temp = (byte)(temp << i);
- temp = (byte) (succeed & temp);
- if(temp != 0 ){
- byte succeedCode = GENE_CODE.getGeneCodeFromBitMap(temp);
- byte[] newKmer = KmerUtil.shiftKmerWithNextCode(KMER_SIZE, kmerValue, 0, kmerLength, succeedCode);
- outputKmer.set(newKmer, 0, kmerLength);
- bitFlag = (byte) 0x80;
- outputAdjList.set(null, 0, 0, (byte)0, bitFlag, KMER_SIZE);
- output.collect(outputKmer, outputAdjList);
- }
- }
- for(int i = 0; i < 4;i ++){
- byte temp = 0x01;
- temp = (byte)(temp << i);
- temp = (byte)(precursor & temp);
- if(temp != 0){
- byte precurCode = GENE_CODE.getGeneCodeFromBitMap(temp);
- byte[] newKmer = KmerUtil.shiftKmerWithPreCode(KMER_SIZE, kmerValue, 0, kmerLength, precurCode);
- outputKmer.set(newKmer, 0, kmerLength);
- bitFlag = (byte)0x40;
- outputAdjList.set(null, 0, 0, (byte)0, bitFlag, KMER_SIZE);
- output.collect(outputKmer, outputAdjList);
- }
- }
- }
- }
-}
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/SNodeInitialReducer.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/SNodeInitialReducer.java
deleted file mode 100644
index 7b918b6..0000000
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/pathmergingh2/SNodeInitialReducer.java
+++ /dev/null
@@ -1,72 +0,0 @@
-package edu.uci.ics.pathmergingh2;
-
-import java.io.IOException;
-import java.util.Iterator;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-
-import edu.uci.ics.genomix.type.KmerUtil;
-
-@SuppressWarnings("deprecation")
-public class SNodeInitialReducer extends MapReduceBase implements
- Reducer<BytesWritable, MergePathValueWritable, BytesWritable, MergePathValueWritable> {
- public BytesWritable outputKmer = new BytesWritable();
- public MergePathValueWritable outputAdjList = new MergePathValueWritable();
- public static int KMER_SIZE;
-
- public void configure(JobConf job) {
- KMER_SIZE = Integer.parseInt(job.get("sizeKmer"));
- }
- @Override
- public void reduce(BytesWritable key, Iterator<MergePathValueWritable> values,
- OutputCollector<BytesWritable, MergePathValueWritable> output, Reporter reporter) throws IOException {
- outputAdjList = values.next();
- outputKmer.set(key);
- byte[] kmerValue = key.getBytes();
- int kmerLength = key.getLength();
- if (values.hasNext() == true) {
- if (outputAdjList.getFlag() == 2) {
- byte bitFlag = 1;
- outputAdjList.set(null, 0, 0, outputAdjList.getAdjBitMap(), bitFlag, outputAdjList.getKmerSize());
- output.collect(outputKmer, outputAdjList);
- } else {
- boolean flag = false;
- while (values.hasNext()) {
- outputAdjList = values.next();
- if (outputAdjList.getFlag() == 2) {
- flag = true;
- break;
- }
- else{
-
- }
- }
- if (flag == true) {
- byte bitFlag = 1;
- outputAdjList.set(null, 0, 0, outputAdjList.getAdjBitMap(), bitFlag, outputAdjList.getKmerSize());
- output.collect(outputKmer, outputAdjList);
- }
- }
- } else {
- if (outputAdjList.getFlag() == 2) {
- byte bitFlag = 0;
- outputAdjList.set(null, 0, 0, outputAdjList.getAdjBitMap(), bitFlag, outputAdjList.getKmerSize());
- output.collect(outputKmer, outputAdjList);
- }
- else {
- byte bitFlag = outputAdjList.getFlag();
- byte adjBitMap = outputAdjList.getAdjBitMap();
- byte precurCode = (byte) ((0xC0 & bitFlag) >> 6);
- bitFlag = 0;
- byte[] newKmer = KmerUtil.shiftKmerWithPreCode(KMER_SIZE, kmerValue, 0, kmerLength, precurCode);
- outputKmer.set(newKmer, 0, kmerLength);
- outputAdjList.set(null, 0, 0, adjBitMap, bitFlag, kmerLength);
- output.collect(outputKmer, outputAdjList);
- }
- }
- }
-}
diff --git a/genomix/genomix-hadoop/src/test/java/edu/uci/ics/pathmergingh2/MergePathH2Test.java b/genomix/genomix-hadoop/src/test/java/edu/uci/ics/pathmergingh2/MergePathH2Test.java
deleted file mode 100644
index 3e66214..0000000
--- a/genomix/genomix-hadoop/src/test/java/edu/uci/ics/pathmergingh2/MergePathH2Test.java
+++ /dev/null
@@ -1,103 +0,0 @@
-package edu.uci.ics.pathmergingh2;
-
-import java.io.BufferedWriter;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MiniMRCluster;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.junit.Test;
-import edu.uci.ics.utils.TestUtils;
-
-@SuppressWarnings("deprecation")
-public class MergePathH2Test {
- private static final String ACTUAL_RESULT_DIR = "actual3";
- private static final String COMPARE_DIR = "compare";
- private JobConf conf = new JobConf();
- private static final String HADOOP_CONF_PATH = ACTUAL_RESULT_DIR + File.separator + "conf.xml";
- private static final String DATA_PATH = "actual2" + "/result2" + "/part-00000";
- private static final String HDFS_PATH = "/webmap";
- private static final String HDFA_PATH_DATA = "/webmapdata";
-
- private static final String RESULT_PATH = "/result3";
- private static final String EXPECTED_PATH = "expected/result3";
- private static final String TEST_SOURCE_DIR = COMPARE_DIR + RESULT_PATH + "/comparesource.txt";
- private static final int COUNT_REDUCER = 4;
- private static final int SIZE_KMER = 3;
-
- private MiniDFSCluster dfsCluster;
- private MiniMRCluster mrCluster;
- private FileSystem dfs;
-
- @SuppressWarnings("resource")
- @Test
- public void test() throws Exception {
- FileUtils.forceMkdir(new File(ACTUAL_RESULT_DIR));
- FileUtils.cleanDirectory(new File(ACTUAL_RESULT_DIR));
- startHadoop();
-
- MergePathH2Driver tldriver = new MergePathH2Driver();
-
- tldriver.run(HDFS_PATH, RESULT_PATH, HDFA_PATH_DATA, COUNT_REDUCER, SIZE_KMER, 3, HADOOP_CONF_PATH);
-
-/* SequenceFile.Reader reader = null;
- Path path = new Path(HDFA_PATH_DATA + "/complete2" + "/complete2-r-00000");
- reader = new SequenceFile.Reader(dfs, path, conf);
- BytesWritable key = (BytesWritable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- MergePathValueWritable value = (MergePathValueWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
- File filePathTo = new File(TEST_SOURCE_DIR);
- BufferedWriter bw = new BufferedWriter(new FileWriter(filePathTo));
- while (reader.next(key, value)) {
- bw.write(key.toString() + "\t" + value.getAdjBitMap() + "\t" + value.getFlag());
- bw.newLine();
- }
- bw.close();
-
- dumpResult();
- TestUtils.compareWithResult(new File(TEST_SOURCE_DIR), new File(EXPECTED_PATH));
-*/
- cleanupHadoop();
-
- }
- private void startHadoop() throws IOException {
- FileSystem lfs = FileSystem.getLocal(new Configuration());
- lfs.delete(new Path("build"), true);
- System.setProperty("hadoop.log.dir", "logs");
- dfsCluster = new MiniDFSCluster(conf, 2, true, null);
- dfs = dfsCluster.getFileSystem();
- mrCluster = new MiniMRCluster(4, dfs.getUri().toString(), 2);
-
- Path src = new Path(DATA_PATH);
- Path dest = new Path(HDFS_PATH + "/");
- dfs.mkdirs(dest);
- dfs.copyFromLocalFile(src, dest);
- Path data = new Path(HDFA_PATH_DATA + "/");
- dfs.mkdirs(data);
-
- DataOutputStream confOutput = new DataOutputStream(new FileOutputStream(new File(HADOOP_CONF_PATH)));
- conf.writeXml(confOutput);
- confOutput.flush();
- confOutput.close();
- }
-
- private void cleanupHadoop() throws IOException {
- mrCluster.shutdown();
- dfsCluster.shutdown();
- }
-
- private void dumpResult() throws IOException {
- Path src = new Path(HDFA_PATH_DATA + "/" + "complete2");
- Path dest = new Path(ACTUAL_RESULT_DIR + "/");
- dfs.copyToLocalFile(src, dest);
- }
-}