add debug code
diff --git a/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/client/Client.java b/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/client/Client.java
index 6c93433..aaa7e53 100644
--- a/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/client/Client.java
+++ b/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/client/Client.java
@@ -45,8 +45,8 @@
@Option(name = "-runtime-profiling", usage = "whether to do runtime profifling", required = false)
public String profiling = "false";
- @Option(name = "-filter-kmer", usage = "whether to do runtime profifling", required = false)
- public String filterKmer = "";
+ //@Option(name = "-filter-kmer", usage = "whether to do runtime profifling", required = false)
+ //public String filterKmer = "";
}
public static void run(String[] args, PregelixJob job) throws Exception {
@@ -73,7 +73,7 @@
job.getConfiguration().setInt(TwoStepLogAlgorithmForPathMergeVertex.ITERATIONS, options.numIteration);
job.getConfiguration().setInt(ThreeStepLogAlgorithmForPathMergeVertex.ITERATIONS, options.numIteration);
}
- job.getConfiguration().set(NaiveFilterVertex.FILTERKMER, options.filterKmer);
+ //job.getConfiguration().set(NaiveFilterVertex.FILTERKMER, options.filterKmer);
return options;
}
diff --git a/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/format/NaiveAlgorithmForPathMergeOutputFormat.java b/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/format/NaiveAlgorithmForPathMergeOutputFormat.java
index 3dd0f2f..e3f14a4 100644
--- a/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/format/NaiveAlgorithmForPathMergeOutputFormat.java
+++ b/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/format/NaiveAlgorithmForPathMergeOutputFormat.java
@@ -8,7 +8,6 @@
import edu.uci.ics.genomix.pregelix.api.io.binary.BinaryVertexOutputFormat;
import edu.uci.ics.genomix.pregelix.io.ValueStateWritable;
-import edu.uci.ics.genomix.pregelix.type.State;
import edu.uci.ics.genomix.type.KmerBytesWritable;
import edu.uci.ics.pregelix.api.graph.Vertex;
import edu.uci.ics.pregelix.api.io.VertexWriter;
@@ -37,9 +36,9 @@
@Override
public void writeVertex(Vertex<KmerBytesWritable, ValueStateWritable, NullWritable, ?> vertex) throws IOException,
InterruptedException {
- if(vertex.getVertexValue().getState() == State.FILTER
- || vertex.getVertexValue().getState() == State.FINAL_VERTEX)
- getRecordWriter().write(vertex.getVertexId(),vertex.getVertexValue());
+ //if(vertex.getVertexValue().getState() == State.FILTER
+ // || vertex.getVertexValue().getState() == State.FINAL_VERTEX)
+ getRecordWriter().write(vertex.getVertexId(),vertex.getVertexValue());
}
}
}
diff --git a/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/operator/NaiveAlgorithmForPathMergeVertex.java b/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/operator/NaiveAlgorithmForPathMergeVertex.java
index b20d6ea..ac52e0d 100644
--- a/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/operator/NaiveAlgorithmForPathMergeVertex.java
+++ b/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/operator/NaiveAlgorithmForPathMergeVertex.java
@@ -52,7 +52,7 @@
* Naive Algorithm for path merge graph
*/
public class NaiveAlgorithmForPathMergeVertex extends Vertex<KmerBytesWritable, ValueStateWritable, NullWritable, NaiveAlgorithmMessageWritable>{
- public static Logger logger = Logger.getLogger(NaiveAlgorithmForPathMergeVertex.class.getName());
+ //public static Logger logger = Logger.getLogger(NaiveAlgorithmForPathMergeVertex.class.getName());
public static final String KMER_SIZE = "NaiveAlgorithmForPathMergeVertex.kmerSize";
@@ -186,12 +186,12 @@
responseMsgToHeadVertex();
}
}
- String log = "Iterator:" + getSuperstep();
+ /*String log = "Iterator:" + getSuperstep();
log += getVertexId().toString() + "\t";
log += getVertexValue().getMergeChain().toString() + "\t";
log += GeneCode.getSymbolFromBitMap(getVertexValue().getAdjMap()) + "\t";
log += getVertexValue().getState();
- logger.info(log);
+ logger.info(log);*/
voteToHalt();
}
diff --git a/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/operator/TwoStepLogAlgorithmForPathMergeVertex.java b/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/operator/TwoStepLogAlgorithmForPathMergeVertex.java
index 2ef2932..eb5edc3 100644
--- a/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/operator/TwoStepLogAlgorithmForPathMergeVertex.java
+++ b/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/operator/TwoStepLogAlgorithmForPathMergeVertex.java
@@ -49,7 +49,7 @@
* The details about message are in edu.uci.ics.pregelix.example.io.MessageWritable.
*/
public class TwoStepLogAlgorithmForPathMergeVertex extends Vertex<KmerBytesWritable, ValueStateWritable, NullWritable, LogAlgorithmMessageWritable>{
- public static Logger logger = Logger.getLogger(TwoStepLogAlgorithmForPathMergeVertex.class.getName());
+ //public static Logger logger = Logger.getLogger(TwoStepLogAlgorithmForPathMergeVertex.class.getName());
public static final String KMER_SIZE = "TwoStepLogAlgorithmForPathMergeVertex.kmerSize";
public static final String ITERATIONS = "TwoStepLogAlgorithmForPathMergeVertex.iteration";
@@ -341,12 +341,12 @@
}
else
voteToHalt();
- String log = "Iterator:" + getSuperstep();
+ /*String log = "Iterator:" + getSuperstep();
log += getVertexId().toString() + "\t";
log += getVertexValue().getMergeChain().toString() + "\t";
log += GeneCode.getSymbolFromBitMap(getVertexValue().getAdjMap()) + "\t";
log += getVertexValue().getState();
- logger.info(log);
+ logger.info(log);*/
}
/**
* @param args
diff --git a/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/sequencefile/GenerateSmallFile.java b/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/sequencefile/GenerateSmallFile.java
index 88bec61..d9fd35f 100644
--- a/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/sequencefile/GenerateSmallFile.java
+++ b/genomix/genomix-pregelix/src/main/java/edu/uci/ics/genomix/pregelix/sequencefile/GenerateSmallFile.java
@@ -1,6 +1,11 @@
package edu.uci.ics.genomix.pregelix.sequencefile;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.FileWriter;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
@@ -15,38 +20,81 @@
public class GenerateSmallFile {
- public static void generateNumOfLinesFromBigFile(Path inFile, Path outFile, int numOfLines) throws IOException{
+ public static void generateNumOfLinesFromGraphBuildResuiltBigFile(
+ Path inFile, Path outFile, int numOfLines) throws IOException {
Configuration conf = new Configuration();
FileSystem fileSys = FileSystem.get(conf);
- SequenceFile.Reader reader = new SequenceFile.Reader(fileSys, inFile, conf);
- SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf,
- outFile, KmerBytesWritable.class, KmerCountValue.class,
- CompressionType.NONE);
- KmerBytesWritable outKey = new KmerBytesWritable(55);
- KmerCountValue outValue = new KmerCountValue();
- int i = 0;
-
- for(i = 0; i < numOfLines; i++){
- //System.out.println(i);
- reader.next(outKey, outValue);
- writer.append(outKey, outValue);
- }
- writer.close();
- reader.close();
+ SequenceFile.Reader reader = new SequenceFile.Reader(fileSys, inFile,
+ conf);
+ SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf,
+ outFile, KmerBytesWritable.class, KmerCountValue.class,
+ CompressionType.NONE);
+ KmerBytesWritable outKey = new KmerBytesWritable(55);
+ KmerCountValue outValue = new KmerCountValue();
+ int i = 0;
+
+ for (i = 0; i < numOfLines; i++) {
+ // System.out.println(i);
+ reader.next(outKey, outValue);
+ writer.append(outKey, outValue);
+ }
+ writer.close();
+ reader.close();
}
- /**
- * @param args
- * @throws IOException
- */
+
+ public static void generateNumOfLinesFromGraphBuildResuiltBigFile(
+ String inFile, String outFile, int numOfLines) throws IOException {
+ String lines = readTextFile(inFile, numOfLines);
+ writeTextFile(outFile, lines);
+ }
+
public static void main(String[] args) throws IOException {
- // TODO Auto-generated method stub
- Path dir = new Path("data/test8m");
+ /*Path dir = new Path("data/test8m");
Path outDir = new Path("data/input/test");
FileUtils.cleanDirectory(new File("data/input/test"));
Path inFile = new Path(dir, "part-0");
Path outFile = new Path(outDir, "part-0-out-100");
- generateNumOfLinesFromBigFile(inFile,outFile,100);
+ generateNumOfLinesFromGraphBuildResuiltBigFile(inFile, outFile, 100);*/
+ String inFile = "data/shortjump_1.head8M.fastq";
+ String outFile = "data/testGeneFile";
+ generateNumOfLinesFromGraphBuildResuiltBigFile(inFile, outFile, 100000);
}
+ public static String readTextFile(String fileName, int numOfLines) {
+ String returnValue = "";
+ FileReader file;
+ String line = "";
+ try {
+ file = new FileReader(fileName);
+ BufferedReader reader = new BufferedReader(file);
+ try {
+ while ((numOfLines > 0) && (line = reader.readLine()) != null) {
+ returnValue += line + "\n";
+ numOfLines--;
+ }
+ } finally {
+ reader.close();
+ }
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException("File not found");
+ } catch (IOException e) {
+ throw new RuntimeException("IO Error occured");
+ }
+ return returnValue;
+
+ }
+
+ public static void writeTextFile(String fileName, String s) {
+ FileWriter output;
+ try {
+ output = new FileWriter(fileName);
+ BufferedWriter writer = new BufferedWriter(output);
+ writer.write(s);
+ writer.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ }
}
diff --git a/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/PathMergeSmallTestSuite.java b/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/PathMergeSmallTestSuite.java
index d05fc7a..898d059 100644
--- a/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/PathMergeSmallTestSuite.java
+++ b/genomix/genomix-pregelix/src/test/java/edu/uci/ics/genomix/pregelix/pathmerge/PathMergeSmallTestSuite.java
@@ -20,6 +20,7 @@
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
+import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -30,6 +31,7 @@
import junit.framework.TestSuite;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -44,15 +46,15 @@
private static final Logger LOGGER = Logger
.getLogger(PathMergeSmallTestSuite.class.getName());
- public static final String PreFix = "data";
+ public static final String PreFix = "graphbuildresult"; //"graphbuildresult";
public static final String[] TestDir = { PreFix + File.separator
- + "test8m"};
+ + "result"};
/*+ "BridgePath", PreFix + File.separator
+ "CyclePath", PreFix + File.separator
+ "SimplePath", PreFix + File.separator
+ "SinglePath", PreFix + File.separator
- + "TreePath"};*/
- /*+ "2", PreFix + File.separator
+ + "TreePath"};
+ + "2"}, PreFix + File.separator
+ "3", PreFix + File.separator
+ "4", PreFix + File.separator
+ "5", PreFix + File.separator
@@ -60,7 +62,6 @@
+ "7", PreFix + File.separator
+ "8", PreFix + File.separator
+ "9", PreFix + File.separator
- + "10", PreFix + File.separator
+ "TwoKmer", PreFix + File.separator
+ "ThreeKmer", PreFix + File.separator
+ "SinglePath", PreFix + File.separator
@@ -112,6 +113,7 @@
File src = new File(testDir);
Path dest = new Path(HDFS_INPUTPATH + File.separator + src.getName());
dfs.mkdirs(dest);
+ //src.listFiles()
//src.listFiles((FilenameFilter)(new WildcardFileFilter("part*")))
for (File f : src.listFiles()){
dfs.copyFromLocalFile(new Path(f.getAbsolutePath()), dest);
diff --git a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/driver/Driver.java b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/driver/Driver.java
index a072b9b..05f1022 100644
--- a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/driver/Driver.java
+++ b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/driver/Driver.java
@@ -130,7 +130,6 @@
do {
start = System.currentTimeMillis();
runLoopBodyIteration(deploymentId, jobGen, i);
- runHDFSWRite(deploymentId, jobGen);
end = System.currentTimeMillis();
time = end - start;
LOG.info("iteration " + i + " finished " + time + "ms");