remove reference code for variable-length lists. We don't keep track of onwership of our data block
diff --git a/genomix/genomix-data/src/main/java/edu/uci/ics/genomix/type/KmerListWritable.java b/genomix/genomix-data/src/main/java/edu/uci/ics/genomix/type/KmerListWritable.java
index b021e1d..cd70bb7 100644
--- a/genomix/genomix-data/src/main/java/edu/uci/ics/genomix/type/KmerListWritable.java
+++ b/genomix/genomix-data/src/main/java/edu/uci/ics/genomix/type/KmerListWritable.java
@@ -33,7 +33,9 @@
}
public KmerListWritable(byte[] data, int offset) {
- setNewReference(data, offset);
+// setNewReference(data, offset);
+ this();
+ setCopy(data, offset);
}
public KmerListWritable(List<KmerBytesWritable> kmers) {
@@ -44,15 +46,15 @@
}
}
- public void setNewReference(byte[] data, int offset) {
- valueCount = Marshal.getInt(data, offset);
- if (valueCount * KmerBytesWritable.getBytesPerKmer() > data.length - offset) {
- throw new IllegalArgumentException("Specified data buffer (len=" + (data.length - offset)
- + ") is not large enough to store requested number of elements (" + valueCount + ")!");
- }
- this.storage = data;
- this.offset = offset;
- }
+// public void setNewReference(byte[] data, int offset) {
+// valueCount = Marshal.getInt(data, offset);
+// if (valueCount * KmerBytesWritable.getBytesPerKmer() > data.length - offset) {
+// throw new IllegalArgumentException("Specified data buffer (len=" + (data.length - offset)
+// + ") is not large enough to store requested number of elements (" + valueCount + ")!");
+// }
+// this.storage = data;
+// this.offset = offset;
+// }
public void append(KmerBytesWritable kmer) {
setSize((1 + valueCount) * KmerBytesWritable.getBytesPerKmer() + HEADER_SIZE);
@@ -121,9 +123,7 @@
}
public void reset() {
- storage = EMPTY_BYTES;
valueCount = 0;
- offset = 0;
}
public KmerBytesWritable getPosition(int i) {
@@ -148,7 +148,7 @@
System.arraycopy(newData, offset + HEADER_SIZE, storage, this.offset + HEADER_SIZE, valueCount
* KmerBytesWritable.getBytesPerKmer());
}
- Marshal.putInt(valueCount, storage, offset);
+ Marshal.putInt(valueCount, storage, this.offset);
}
@Override
diff --git a/genomix/genomix-data/src/main/java/edu/uci/ics/genomix/type/NodeWritable.java b/genomix/genomix-data/src/main/java/edu/uci/ics/genomix/type/NodeWritable.java
index 81c9c92..284d417 100644
--- a/genomix/genomix-data/src/main/java/edu/uci/ics/genomix/type/NodeWritable.java
+++ b/genomix/genomix-data/src/main/java/edu/uci/ics/genomix/type/NodeWritable.java
@@ -1,25 +1,27 @@
package edu.uci.ics.genomix.type;
+import java.io.ByteArrayOutputStream;
import java.io.DataInput;
import java.io.DataOutput;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.io.Serializable;
+import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.WritableComparable;
public class NodeWritable implements WritableComparable<NodeWritable>, Serializable{
private static final long serialVersionUID = 1L;
- public static final NodeWritable EMPTY_NODE = new NodeWritable(0);
+ public static final NodeWritable EMPTY_NODE = new NodeWritable();
private PositionListWritable nodeIdList;
private KmerListWritable forwardForwardList;
private KmerListWritable forwardReverseList;
private KmerListWritable reverseForwardList;
private KmerListWritable reverseReverseList;
- private KmerBytesWritable kmer;
- private int kmerlength = 0;
+ private VKmerBytesWritable kmer;
// merge/update directions
public static class DirectionFlag {
@@ -31,48 +33,41 @@
}
public NodeWritable() {
- this(0);
- }
-
- public NodeWritable(int kmerlenth) {
- this.kmerlength = kmerlenth;
nodeIdList = new PositionListWritable();
- forwardForwardList = new KmerListWritable(kmerlenth);
- forwardReverseList = new KmerListWritable(kmerlenth);
- reverseForwardList = new KmerListWritable(kmerlenth);
- reverseReverseList = new KmerListWritable(kmerlenth);
- kmer = new KmerBytesWritable(); //in graph construction - not set kmerlength Optimization: VKmer
+ forwardForwardList = new KmerListWritable();
+ forwardReverseList = new KmerListWritable();
+ reverseForwardList = new KmerListWritable();
+ reverseReverseList = new KmerListWritable();
+ kmer = new VKmerBytesWritable(); // in graph construction - not set kmerlength Optimization: VKmer
}
public NodeWritable(PositionListWritable nodeIdList, KmerListWritable FFList, KmerListWritable FRList,
- KmerListWritable RFList, KmerListWritable RRList, KmerBytesWritable kmer) {
- this(kmer.getKmerLength());
+ KmerListWritable RFList, KmerListWritable RRList, VKmerBytesWritable kmer) {
+ this();
set(nodeIdList, FFList, FRList, RFList, RRList, kmer);
}
public void set(NodeWritable node){
- this.kmerlength = node.kmerlength;
set(node.nodeIdList, node.forwardForwardList, node.forwardReverseList, node.reverseForwardList,
node.reverseReverseList, node.kmer);
}
public void set(PositionListWritable nodeIdList, KmerListWritable FFList, KmerListWritable FRList,
- KmerListWritable RFList, KmerListWritable RRList, KmerBytesWritable kmer) {
+ KmerListWritable RFList, KmerListWritable RRList, VKmerBytesWritable kmer2) {
this.nodeIdList.set(nodeIdList);
this.forwardForwardList.setCopy(FFList);
this.forwardReverseList.setCopy(FRList);
this.reverseForwardList.setCopy(RFList);
this.reverseReverseList.setCopy(RRList);
- this.kmer.setAsCopy(kmer);
+ this.kmer.setAsCopy(kmer2);
}
- public void reset(int kmerSize) {
- this.kmerlength = kmerSize;
+ public void reset() {
this.nodeIdList.reset();
- this.forwardForwardList.reset(kmerSize);
- this.forwardReverseList.reset(kmerSize);
- this.reverseForwardList.reset(kmerSize);
- this.reverseReverseList.reset(kmerSize);
+ this.forwardForwardList.reset();
+ this.forwardReverseList.reset();
+ this.reverseForwardList.reset();
+ this.reverseReverseList.reset();
this.kmer.reset(0);
}
@@ -85,24 +80,16 @@
this.nodeIdList.set(nodeIdList);
}
- public KmerBytesWritable getKmer() {
+ public VKmerBytesWritable getKmer() {
return kmer;
}
- public void setKmer(KmerBytesWritable kmer) {
+ public void setKmer(VKmerBytesWritable kmer) {
this.kmer.setAsCopy(kmer);
}
- public int getKmerlength() {
- return kmerlength;
- }
-
- public void setKmerlength(int kmerlength) {
- this.kmerlength = kmerlength;
- }
-
- public int getCount() {
- return kmer.getKmerLength();
+ public int getKmerLength() {
+ return kmer.getKmerLetterLength();
}
public KmerListWritable getFFList() {
@@ -154,7 +141,6 @@
@Override
public void write(DataOutput out) throws IOException {
- out.writeInt(kmerlength);
this.nodeIdList.write(out);
this.forwardForwardList.write(out);
this.forwardReverseList.write(out);
@@ -165,8 +151,7 @@
@Override
public void readFields(DataInput in) throws IOException {
- this.kmerlength = in.readInt();
- reset(kmerlength);
+ reset();
this.nodeIdList.readFields(in);
this.forwardForwardList.readFields(in);
this.forwardReverseList.readFields(in);
@@ -211,13 +196,13 @@
return sbuilder.toString();
}
- public void mergeForwardNext(NodeWritable nextNode, int initialKmerSize) {
+ public void mergeForwardNext(final NodeWritable nextNode, int initialKmerSize) {
this.forwardForwardList.setCopy(nextNode.forwardForwardList);
this.forwardReverseList.setCopy(nextNode.forwardReverseList);
kmer.mergeWithFFKmer(initialKmerSize, nextNode.getKmer());
}
- public void mergeForwardPre(NodeWritable preNode, int initialKmerSize) {
+ public void mergeForwardPre(final NodeWritable preNode, int initialKmerSize) {
this.reverseForwardList.setCopy(preNode.reverseForwardList);
this.reverseReverseList.setCopy(preNode.reverseReverseList);
kmer.mergeWithRRKmer(initialKmerSize, preNode.getKmer());
diff --git a/genomix/genomix-data/src/main/java/edu/uci/ics/genomix/type/PositionListWritable.java b/genomix/genomix-data/src/main/java/edu/uci/ics/genomix/type/PositionListWritable.java
index b056c14..a7d67d1 100644
--- a/genomix/genomix-data/src/main/java/edu/uci/ics/genomix/type/PositionListWritable.java
+++ b/genomix/genomix-data/src/main/java/edu/uci/ics/genomix/type/PositionListWritable.java
@@ -13,78 +13,81 @@
import edu.uci.ics.genomix.data.Marshal;
import edu.uci.ics.genomix.type.PositionWritable;
-public class PositionListWritable implements Writable, Iterable<PositionWritable>, Serializable{
+public class PositionListWritable implements Writable, Iterable<PositionWritable>, Serializable {
private static final long serialVersionUID = 1L;
protected byte[] storage;
protected int offset;
protected int valueCount;
- protected static final byte[] EMPTY = {};
-
+ protected static final byte[] EMPTY_BYTES = {0,0,0,0};
+ protected static final int HEADER_SIZE = 4;
+
protected PositionWritable posIter = new PositionWritable();
-
+
public PositionListWritable() {
- this.storage = EMPTY;
+ this.storage = EMPTY_BYTES;
this.valueCount = 0;
this.offset = 0;
}
-
- public PositionListWritable(int count, byte[] data, int offset) {
- setNewReference(count, data, offset);
+
+ public PositionListWritable(byte[] data, int offset) {
+// setNewReference(data, offset);
+ set(data, offset);
}
-
+
public PositionListWritable(List<PositionWritable> posns) {
this();
- setSize(posns.size()); // reserve space for all elements
+ setSize(posns.size()); // reserve space for all elements
for (PositionWritable p : posns) {
append(p);
}
}
-
- public void setNewReference(int count, byte[] data, int offset) {
- this.valueCount = count;
- this.storage = data;
- this.offset = offset;
- }
-
+
+// public void setNewReference(byte[] data, int offset) {
+// this.valueCount = Marshal.getInt(data, offset);
+// this.storage = data;
+// this.offset = offset;
+// isReference = true;
+// }
+
public void append(long uuid) {
- setSize((1 + valueCount) * PositionWritable.LENGTH);
- Marshal.putLong(uuid, storage, offset + valueCount * PositionWritable.LENGTH);
+ setSize((1 + valueCount) * PositionWritable.LENGTH + HEADER_SIZE);
+ Marshal.putLong(uuid, storage, offset + valueCount * PositionWritable.LENGTH + HEADER_SIZE);
valueCount += 1;
+ Marshal.putInt(valueCount, storage, offset);
}
-
- public void append(byte mateId, long readId, int posId){
+
+ public void append(byte mateId, long readId, int posId) {
append(PositionWritable.makeUUID(mateId, readId, posId));
}
-
+
public void append(PositionWritable pos) {
- if(pos != null)
+ if (pos != null)
append(pos.getUUID());
else
throw new RuntimeException("This position is null pointer!");
}
-
+
/*
* Append the otherList to the end of myList
*/
public void appendList(PositionListWritable otherList) {
if (otherList.valueCount > 0) {
- setSize((valueCount + otherList.valueCount) * PositionWritable.LENGTH);
+ setSize((valueCount + otherList.valueCount) * PositionWritable.LENGTH + HEADER_SIZE);
// copy contents of otherList into the end of my storage
- System.arraycopy(otherList.storage, otherList.offset,
- storage, offset + valueCount * PositionWritable.LENGTH,
- otherList.valueCount * PositionWritable.LENGTH);
+ System.arraycopy(otherList.storage, otherList.offset + HEADER_SIZE, storage, offset + valueCount
+ * PositionWritable.LENGTH + HEADER_SIZE, otherList.valueCount * PositionWritable.LENGTH);
valueCount += otherList.valueCount;
+ Marshal.putInt(valueCount, storage, offset);
}
}
-
+
/**
* Save the union of my list and otherList. Uses a temporary HashSet for
* uniquefication
*/
public void unionUpdate(PositionListWritable otherList) {
int newSize = valueCount + otherList.valueCount;
- HashSet<PositionWritable> uniqueElements = new HashSet<PositionWritable>(
- newSize);
+ HashSet<PositionWritable> uniqueElements = new HashSet<PositionWritable>(newSize);
for (PositionWritable pos : this) {
uniqueElements.add(pos);
}
@@ -92,42 +95,47 @@
uniqueElements.add(pos);
}
valueCount = 0;
- setSize(newSize);
+ setSize(newSize * PositionWritable.LENGTH + HEADER_SIZE);
for (PositionWritable pos : uniqueElements) {
append(pos);
}
}
-
+
public static int getCountByDataLength(int length) {
if (length % PositionWritable.LENGTH != 0) {
throw new IllegalArgumentException("Length of positionlist is invalid");
}
return length / PositionWritable.LENGTH;
}
-
+
public void set(PositionListWritable otherList) {
- set(otherList.valueCount, otherList.storage, otherList.offset);
+ set(otherList.storage, otherList.offset);
}
- public void set(int valueCount, byte[] newData, int offset) {
- this.valueCount = valueCount;
- setSize(valueCount * PositionWritable.LENGTH);
+ public void set(byte[] newData, int offset) {
+ valueCount = Marshal.getInt(newData, offset);
+ setSize(valueCount * PositionWritable.LENGTH + HEADER_SIZE);
if (valueCount > 0) {
- System.arraycopy(newData, offset, storage, this.offset, valueCount * PositionWritable.LENGTH);
+ System.arraycopy(newData, offset + HEADER_SIZE, storage, this.offset + HEADER_SIZE, valueCount * PositionWritable.LENGTH);
}
+ Marshal.putInt(valueCount, storage, offset);
}
public void reset() {
valueCount = 0;
+ Marshal.putInt(valueCount, storage, offset);
}
-
+
protected void setSize(int size) {
if (size > getCapacity()) {
setCapacity((size * 3 / 2));
}
}
-
+
protected int getCapacity() {
+// if (isReference) { // my storage is a borrowed reference so I can't expand beyond my original size
+// return valueCount * PositionWritable.LENGTH + HEADER_SIZE;
+// }
return storage.length - offset;
}
@@ -141,7 +149,7 @@
offset = 0;
}
}
-
+
public PositionWritable getPosition(int i) {
if (i >= valueCount) {
throw new ArrayIndexOutOfBoundsException("No such positions");
@@ -149,14 +157,14 @@
posIter.setNewReference(storage, offset + i * PositionWritable.LENGTH);
return posIter;
}
-
+
public void resetPosition(int i, long uuid) {
if (i >= valueCount) {
throw new ArrayIndexOutOfBoundsException("No such positions");
}
Marshal.putLong(uuid, storage, offset + i * PositionWritable.LENGTH);
}
-
+
public int getCountOfPosition() {
return valueCount;
}
@@ -172,7 +180,7 @@
public int getLength() {
return valueCount * PositionWritable.LENGTH;
}
-
+
@Override
public Iterator<PositionWritable> iterator() {
Iterator<PositionWritable> it = new Iterator<PositionWritable>() {
@@ -191,50 +199,51 @@
@Override
public void remove() {
- if(currentIndex < valueCount)
- System.arraycopy(storage, offset + currentIndex * PositionWritable.LENGTH,
- storage, offset + (currentIndex - 1) * PositionWritable.LENGTH,
- (valueCount - currentIndex) * PositionWritable.LENGTH);
+ if (currentIndex < valueCount)
+ System.arraycopy(storage, offset + currentIndex * PositionWritable.LENGTH, storage, offset
+ + (currentIndex - 1) * PositionWritable.LENGTH, (valueCount - currentIndex)
+ * PositionWritable.LENGTH);
valueCount--;
currentIndex--;
}
};
return it;
}
-
+
/*
* remove the first instance of @toRemove. Uses a linear scan. Throws an exception if not in this list.
*/
public void remove(PositionWritable toRemove, boolean ignoreMissing) {
Iterator<PositionWritable> posIterator = this.iterator();
while (posIterator.hasNext()) {
- if(toRemove.equals(posIterator.next())) {
+ if (toRemove.equals(posIterator.next())) {
posIterator.remove();
return;
}
}
if (!ignoreMissing) {
- throw new ArrayIndexOutOfBoundsException("the PositionWritable `" + toRemove.toString() + "` was not found in this list.");
+ throw new ArrayIndexOutOfBoundsException("the PositionWritable `" + toRemove.toString()
+ + "` was not found in this list.");
}
}
-
+
public void remove(PositionWritable toRemove) {
- remove(toRemove, false);
+ remove(toRemove, false);
}
-
+
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(valueCount);
out.write(storage, offset, valueCount * PositionWritable.LENGTH);
}
-
+
@Override
public void readFields(DataInput in) throws IOException {
this.valueCount = in.readInt();
setSize(valueCount * PositionWritable.LENGTH);
in.readFully(storage, offset, valueCount * PositionWritable.LENGTH);
}
-
+
@Override
public String toString() {
StringBuilder sbuilder = new StringBuilder();
@@ -250,12 +259,12 @@
}
return sbuilder.toString();
}
-
+
@Override
public int hashCode() {
return Marshal.hashBytes(getByteArray(), getStartOffset(), getLength());
}
-
+
@Override
public boolean equals(Object o) {
if (!(o instanceof PositionListWritable))
@@ -263,9 +272,9 @@
PositionListWritable other = (PositionListWritable) o;
if (this.valueCount != other.valueCount)
return false;
- for (int i=0; i < this.valueCount; i++) {
- if (!this.getPosition(i).equals(other.getPosition(i)))
- return false;
+ for (int i = 0; i < this.valueCount; i++) {
+ if (!this.getPosition(i).equals(other.getPosition(i)))
+ return false;
}
return true;
}
diff --git a/genomix/genomix-data/src/test/java/edu/uci/ics/genomix/data/test/KmerListWritableTest.java b/genomix/genomix-data/src/test/java/edu/uci/ics/genomix/data/test/KmerListWritableTest.java
index 6e95653..5a69a3c 100644
--- a/genomix/genomix-data/src/test/java/edu/uci/ics/genomix/data/test/KmerListWritableTest.java
+++ b/genomix/genomix-data/src/test/java/edu/uci/ics/genomix/data/test/KmerListWritableTest.java
@@ -32,9 +32,9 @@
}
kmerList.reset();
+ KmerBytesWritable.setGlobalKmerLength(5);
//add one more kmer each time and fix kmerSize
for (int i = 0; i < 200; i++) {
- KmerBytesWritable.setGlobalKmerLength(5);
kmer = new KmerBytesWritable();
String randomString = generateString(5);
byte[] array = randomString.getBytes();