svn merge -r2972:3096 https://hyracks.googlecode.com/svn/branches/fullstack_asterix_stabilization
git-svn-id: https://hyracks.googlecode.com/svn/branches/fullstack_genomix@3099 123451ca-8445-de46-9d55-352943316053
diff --git a/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/data/partition/KmerHashPartitioncomputerFactory.java b/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/data/partition/KmerHashPartitioncomputerFactory.java
index 0f8b4a9..d56c7c2 100644
--- a/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/data/partition/KmerHashPartitioncomputerFactory.java
+++ b/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/data/partition/KmerHashPartitioncomputerFactory.java
@@ -35,11 +35,11 @@
ByteBuffer buf = accessor.getBuffer();
- int part = hashBytes(buf.array(), startOffset + fieldOffset + slotLength, fieldLength) % nParts;
- if (part < 0){
- part = -part;
+ int hash = hashBytes(buf.array(), startOffset + fieldOffset + slotLength, fieldLength);
+ if (hash < 0){
+ hash = - (hash+1);
}
- return part;
+ return hash % nParts;
}
};
}
diff --git a/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/dataflow/KMerSequenceWriterFactory.java b/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/dataflow/KMerSequenceWriterFactory.java
index 02be7d8..3cc4b8a 100644
--- a/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/dataflow/KMerSequenceWriterFactory.java
+++ b/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/dataflow/KMerSequenceWriterFactory.java
@@ -11,6 +11,7 @@
import org.apache.hadoop.mapred.JobConf;
import edu.uci.ics.genomix.type.KmerCountValue;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.hdfs.api.ITupleWriter;
@@ -44,11 +45,6 @@
public void write(DataOutput output, ITupleReference tuple)
throws HyracksDataException {
try {
- if (writer == null) {
- writer = SequenceFile.createWriter(cf.getConf(),
- (FSDataOutputStream) output, BytesWritable.class,
- KmerCountValue.class, CompressionType.NONE, null);
- }
byte[] kmer = tuple.getFieldData(0);
int keyStart = tuple.getFieldStart(0);
int keyLength = tuple.getFieldLength(0);
@@ -64,10 +60,26 @@
throw new HyracksDataException(e);
}
}
+ @Override
+ public void open(DataOutput output) throws HyracksDataException {
+ try {
+ writer = SequenceFile.createWriter(cf.getConf(),
+ (FSDataOutputStream) output, BytesWritable.class,
+ KmerCountValue.class, CompressionType.NONE, null);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ @Override
+ public void close(DataOutput output) throws HyracksDataException {
+ // TODO Auto-generated method stub
+
+ }
}
@Override
- public ITupleWriter getTupleWriter() {
+ public ITupleWriter getTupleWriter(IHyracksTaskContext ctx)
+ throws HyracksDataException {
return new TupleWriter(confFactory);
}
diff --git a/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/dataflow/KMerTextWriterFactory.java b/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/dataflow/KMerTextWriterFactory.java
index 332b7d7..b93e723 100644
--- a/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/dataflow/KMerTextWriterFactory.java
+++ b/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/dataflow/KMerTextWriterFactory.java
@@ -4,6 +4,7 @@
import java.io.IOException;
import edu.uci.ics.genomix.type.Kmer;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.hdfs.api.ITupleWriter;
@@ -40,10 +41,24 @@
throw new HyracksDataException(e);
}
}
+
+ @Override
+ public void open(DataOutput output) throws HyracksDataException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void close(DataOutput output) throws HyracksDataException {
+ // TODO Auto-generated method stub
+
+ }
}
@Override
- public ITupleWriter getTupleWriter() {
+ public ITupleWriter getTupleWriter(IHyracksTaskContext ctx)
+ throws HyracksDataException {
+ // TODO Auto-generated method stub
return new TupleWriter();
}
diff --git a/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/dataflow/ReadsKeyValueParserFactory.java b/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/dataflow/ReadsKeyValueParserFactory.java
index 1f0cd73..fe22251 100644
--- a/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/dataflow/ReadsKeyValueParserFactory.java
+++ b/genomix/genomix-hyracks/src/main/java/edu/uci/ics/genomix/dataflow/ReadsKeyValueParserFactory.java
@@ -54,11 +54,6 @@
}
}
- @Override
- public void flush(IFrameWriter writer) throws HyracksDataException {
- FrameUtils.flushFrame(outputBuffer, writer);
- }
-
private void SplitReads(byte[] array, IFrameWriter writer) {
/** first kmer */
byte[] kmer = Kmer.CompressKmer(k, array, 0);
@@ -106,6 +101,17 @@
throw new IllegalStateException(e);
}
}
+
+ @Override
+ public void open(IFrameWriter writer) throws HyracksDataException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void close(IFrameWriter writer) throws HyracksDataException {
+ FrameUtils.flushFrame(outputBuffer, writer);
+ }
};
}
diff --git a/genomix/genomix-hyracks/src/main/resources/conf/cluster.properties b/genomix/genomix-hyracks/src/main/resources/conf/cluster.properties
index 2d0eb26..66251be 100644
--- a/genomix/genomix-hyracks/src/main/resources/conf/cluster.properties
+++ b/genomix/genomix-hyracks/src/main/resources/conf/cluster.properties
@@ -7,7 +7,7 @@
#The directory of hyracks binaries
HYRACKS_HOME="../../../../hyracks"
-WORKPATH="/mnt/data/sda/space/jjia/hyracks"
+WORKPATH=""
#The tmp directory for cc to install jars
CCTMP_DIR=${WORKPATH}/tmp/t1
@@ -21,7 +21,7 @@
NCLOGS_DIR=$NCTMP_DIR/logs
#Comma separated I/O directories for the spilling of external sort
-IO_DIRS="/mnt/data/sda/space/jjia/hyracks/tmp/t3,/mnt/data/sdb/space/jjia/hyracks/tmp/t3,/mnt/data/sdc/space/jjia/hyracks/tmp/t3,/mnt/data/sdd/space/jjia/hyracks/tmp/t3"
+IO_DIRS="${WORKPATH}/tmp/t3"
#The JAVA_HOME
JAVA_HOME=$JAVA_HOME
@@ -33,9 +33,9 @@
FRAME_SIZE=65536
#CC JAVA_OPTS
-CCJAVA_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,address=7001,server=y,suspend=n -Xmx1g -Djava.util.logging.config.file=logging.properties"
+CCJAVA_OPTS="-Xrunjdwp:transport=dt_socket,address=7001,server=y,suspend=n -Xmx1g -Djava.util.logging.config.file=logging.properties"
# Yourkit option: -agentpath:/grid/0/dev/vborkar/tools/yjp-10.0.4/bin/linux-x86-64/libyjpagent.so=port=20001"
#NC JAVA_OPTS
-NCJAVA_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,address=7002,server=y,suspend=n -Xmx10g -Djava.util.logging.config.file=logging.properties"
+NCJAVA_OPTS="-Xrunjdwp:transport=dt_socket,address=7002,server=y,suspend=n -Xmx10g -Djava.util.logging.config.file=logging.properties"
diff --git a/genomix/genomix-hyracks/src/main/resources/conf/master b/genomix/genomix-hyracks/src/main/resources/conf/master
index d71f20c..2fbb50c 100644
--- a/genomix/genomix-hyracks/src/main/resources/conf/master
+++ b/genomix/genomix-hyracks/src/main/resources/conf/master
@@ -1 +1 @@
-asterix-master
+localhost
diff --git a/genomix/genomix-hyracks/src/main/resources/conf/slaves b/genomix/genomix-hyracks/src/main/resources/conf/slaves
index d5272bc..2fbb50c 100644
--- a/genomix/genomix-hyracks/src/main/resources/conf/slaves
+++ b/genomix/genomix-hyracks/src/main/resources/conf/slaves
@@ -1,4 +1 @@
-asterix-001
-asterix-003
-asterix-002
-asterix-004
+localhost