Bugfixes related to combining compressed BTree leaf frames with TypeAwareTupleWriter.
git-svn-id: https://hyracks.googlecode.com/svn/trunk@229 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeLeafFrame.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
index 3802710..c2dd1f4 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
@@ -21,4 +21,6 @@
public void setPrevLeaf(int prevPage);
public int getPrevLeaf();
+
+ public IBTreeTupleReference createTupleReference();
}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeTupleReference.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeTupleReference.java
index 4e82bed..bbeb6a6 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeTupleReference.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeTupleReference.java
@@ -5,7 +5,8 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
public interface IBTreeTupleReference extends ITupleReference {
- public void setFieldCount(int fieldCount);
+ public void setFieldCount(int fieldCount);
+ public void setFieldCount(int fieldStartIndex, int fieldCount);
public void resetByOffset(ByteBuffer buf, int tupleStartOffset);
public void resetByTupleIndex(IBTreeFrame frame, int tupleIndex);
}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java
index 953152e..ecc177d 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java
@@ -129,9 +129,7 @@
else {
newPrefixSlots = new int[totalSlotsNeeded];
}
-
- //System.out.println("TOTALPREFIXBYTES: " + totalPrefixBytes);
-
+
int[] newTupleSlots = new int[tupleCount];
// WARNING: our hope is that compression is infrequent
@@ -149,16 +147,16 @@
int prefixTupleIndex = 0;
uncompressedTupleCount = 0;
- TypeAwareTupleWriter tupleWriter = new TypeAwareTupleWriter(typeTraits);
+ TypeAwareTupleWriter tupleWriter = new TypeAwareTupleWriter(typeTraits);
FieldPrefixTupleReference tupleToWrite = new FieldPrefixTupleReference(tupleWriter.createTupleReference());
tupleToWrite.setFieldCount(fieldCount);
while(tupleIndex < tupleCount) {
- if(kpIndex < keyPartitions.size()) {
+ if(kpIndex < keyPartitions.size()) {
// beginning of keyPartition found, compress entire keyPartition
- if(tupleIndex == keyPartitions.get(kpIndex).firstTupleIndex) {
+ if(tupleIndex == keyPartitions.get(kpIndex).firstTupleIndex) {
// number of fields we decided to use for compression of this keyPartition
int numFieldsToCompress = keyPartitions.get(kpIndex).maxPmiIndex + 1;
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/FieldPrefixNSMLeafFrame.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/FieldPrefixNSMLeafFrame.java
index ee77467..7642638 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/FieldPrefixNSMLeafFrame.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/FieldPrefixNSMLeafFrame.java
@@ -28,6 +28,7 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeFrame;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeTupleReference;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeTupleWriter;
import edu.uci.ics.hyracks.storage.am.btree.api.IFrameCompressor;
import edu.uci.ics.hyracks.storage.am.btree.api.IPrefixSlotManager;
@@ -318,7 +319,7 @@
frameTuple.setFieldCount(fields.length);
for(int i = 0; i < tupleCount; i++) {
frameTuple.resetByTupleIndex(this, i);
- for(int j = 0; j < cmp.getKeyFieldCount(); j++) {
+ for(int j = 0; j < cmp.getKeyFieldCount(); j++) {
ByteArrayInputStream inStream = new ByteArrayInputStream(frameTuple.getFieldData(j), frameTuple.getFieldStart(j), frameTuple.getFieldLength(j));
DataInput dataIn = new DataInputStream(inStream);
Object o = fields[j].deserialize(dataIn);
@@ -532,7 +533,7 @@
// compact both pages
compact(cmp);
rightFrame.compact(cmp);
-
+
// insert last key
targetFrame.insert(tuple, cmp);
@@ -540,9 +541,10 @@
frameTuple.resetByTupleIndex(this, getTupleCount()-1);
int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0, cmp.getKeyFieldCount());
- splitKey.initData(splitKeySize);
+ splitKey.initData(splitKeySize);
tupleWriter.writeTupleFields(frameTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer(), 0);
-
+ splitKey.getTuple().resetByOffset(splitKey.getBuffer(), 0);
+
return 0;
}
@@ -601,4 +603,9 @@
public IBTreeTupleWriter getTupleWriter() {
return tupleWriter;
}
+
+ @Override
+ public IBTreeTupleReference createTupleReference() {
+ return new FieldPrefixTupleReference(tupleWriter.createTupleReference());
+ }
}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/NSMLeafFrame.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/NSMLeafFrame.java
index 363ab23..0fa8c2d 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/NSMLeafFrame.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/NSMLeafFrame.java
@@ -20,6 +20,7 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeFrame;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeTupleReference;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeTupleWriter;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeException;
import edu.uci.ics.hyracks.storage.am.btree.impls.FindSlotMode;
@@ -155,9 +156,9 @@
int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0, cmp.getKeyFieldCount());
splitKey.initData(splitKeySize);
- tupleWriter.writeTupleFields(frameTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer(), 0);
+ tupleWriter.writeTupleFields(frameTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer(), 0);
+ splitKey.getTuple().resetByOffset(splitKey.getBuffer(), 0);
- splitKey.getTuple().resetByOffset(splitKey.getBuffer(), 0);
return 0;
}
@@ -166,4 +167,9 @@
buf.putInt(freeSpaceOff, nextLeafOff + 4);
buf.putInt(totalFreeSpaceOff, buf.capacity() - (nextLeafOff + 4));
}
+
+ @Override
+ public IBTreeTupleReference createTupleReference() {
+ return tupleWriter.createTupleReference();
+ }
}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
index a4effb7..3bfc826 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
@@ -585,6 +585,7 @@
usefulCompression++;
}
else {
+
uselessCompressionTime += (end - start);
uselessCompression++;
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixTupleReference.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixTupleReference.java
index ba17ce7..d818e0a 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixTupleReference.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixTupleReference.java
@@ -46,6 +46,11 @@
}
@Override
+ public void setFieldCount(int fieldStartIndex, int fieldCount) {
+ // not implemented
+ }
+
+ @Override
public int getFieldCount() {
return fieldCount;
}
@@ -63,7 +68,7 @@
return helperTuple.getFieldLength(fIdx);
}
else {
- helperTuple.setFieldCount(fieldCount - numPrefixFields);
+ helperTuple.setFieldCount(numPrefixFields, fieldCount - numPrefixFields);
helperTuple.resetByOffset(frame.getBuffer(), suffixTupleStartOff);
return helperTuple.getFieldLength(fIdx - numPrefixFields);
}
@@ -77,7 +82,7 @@
return helperTuple.getFieldStart(fIdx);
}
else {
- helperTuple.setFieldCount(fieldCount - numPrefixFields);
+ helperTuple.setFieldCount(numPrefixFields, fieldCount - numPrefixFields);
helperTuple.resetByOffset(frame.getBuffer(), suffixTupleStartOff);
return helperTuple.getFieldStart(fIdx - numPrefixFields);
}
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/RangeSearchCursor.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/RangeSearchCursor.java
index b451046..9970114 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/RangeSearchCursor.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/RangeSearchCursor.java
@@ -37,7 +37,7 @@
public RangeSearchCursor(IBTreeLeafFrame frame) {
this.frame = frame;
- this.frameTuple = frame.getTupleWriter().createTupleReference();
+ this.frameTuple = frame.createTupleReference();
}
@Override
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tuples/SimpleTupleReference.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tuples/SimpleTupleReference.java
index 5a7ce72..284efc6 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tuples/SimpleTupleReference.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tuples/SimpleTupleReference.java
@@ -8,6 +8,7 @@
public class SimpleTupleReference implements IBTreeTupleReference {
protected ByteBuffer buf;
+ protected int fieldStartIndex;
protected int fieldCount;
protected int tupleStartOff;
protected int nullFlagsBytes;
@@ -29,6 +30,13 @@
this.fieldCount = fieldCount;
nullFlagsBytes = getNullFlagsBytes();
fieldSlotsBytes = getFieldSlotsBytes();
+ fieldStartIndex = 0;
+ }
+
+ @Override
+ public void setFieldCount(int fieldStartIndex, int fieldCount) {
+ this.fieldCount = fieldCount;
+ this.fieldStartIndex = fieldStartIndex;
}
@Override
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tuples/TypeAwareTupleReference.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tuples/TypeAwareTupleReference.java
index 5f8b6d8..1012800 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tuples/TypeAwareTupleReference.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tuples/TypeAwareTupleReference.java
@@ -8,6 +8,7 @@
public class TypeAwareTupleReference implements IBTreeTupleReference {
protected ByteBuffer buf;
+ protected int fieldStartIndex;
protected int fieldCount;
protected int tupleStartOff;
protected int nullFlagsBytes;
@@ -19,6 +20,7 @@
public TypeAwareTupleReference(ITypeTrait[] typeTraits) {
this.typeTraits = typeTraits;
+ this.fieldStartIndex = 0;
}
@Override
@@ -29,8 +31,9 @@
// decode field slots
int field = 0;
int cumul = 0;
+ int end = fieldStartIndex + fieldCount;
encDec.reset(buf.array(), tupleStartOff + nullFlagsBytes);
- for(int i = 0; i < fieldCount; i++) {
+ for(int i = fieldStartIndex; i < end; i++) {
int staticDataLen = typeTraits[i].getStaticallyKnownDataLength();
if(staticDataLen == ITypeTrait.VARIABLE_LENGTH) {
cumul += encDec.decode();
@@ -49,6 +52,7 @@
resetByOffset(frame.getBuffer(), frame.getTupleOffset(tupleIndex));
}
+ @Override
public void setFieldCount(int fieldCount) {
this.fieldCount = fieldCount;
if(decodedFieldSlots == null) {
@@ -60,6 +64,13 @@
}
}
nullFlagsBytes = getNullFlagsBytes();
+ this.fieldStartIndex = 0;
+ }
+
+ @Override
+ public void setFieldCount(int fieldStartIndex, int fieldCount) {
+ setFieldCount(fieldCount);
+ this.fieldStartIndex = fieldStartIndex;
}
@Override
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tuples/TypeAwareTupleWriter.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tuples/TypeAwareTupleWriter.java
index e82e07f..e47bf2f 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tuples/TypeAwareTupleWriter.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tuples/TypeAwareTupleWriter.java
@@ -123,4 +123,8 @@
public ITypeTrait[] getTypeTraits() {
return typeTraits;
}
+
+ public void setTypeTraits(ITypeTrait[] typeTraits) {
+ this.typeTraits = typeTraits;
+ }
}
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/ListResultCursor.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/ListResultCursor.java
index 44bf9f5..88feb23 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/ListResultCursor.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/ListResultCursor.java
@@ -9,7 +9,7 @@
private List<ByteBuffer> resultBuffers;
private int numResultBuffers;
- private int currentPos = -1;
+ private int currentPos = 0;
public void setResults(List<ByteBuffer> resultBuffers, int numResultBuffers) {
this.resultBuffers = resultBuffers;
@@ -19,8 +19,8 @@
@Override
public boolean hasNext() {
- if(currentPos < numResultBuffers) return true;
- else return false;
+ if(currentPos+1 < numResultBuffers) return true;
+ else return false;
}
@Override
@@ -36,5 +36,5 @@
@Override
public void reset() {
currentPos = -1;
- }
+ }
}
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/SimpleConjunctiveSearcher.java b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/SimpleConjunctiveSearcher.java
index e3067a3..bb13d94 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/SimpleConjunctiveSearcher.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/SimpleConjunctiveSearcher.java
@@ -141,7 +141,7 @@
btree.search(btreeCursor, pred, opCtx);
while(btreeCursor.hasNext()) {
btreeCursor.next();
- maxResultBufIdx = appendTupleToNewResults(btreeCursor, maxResultBufIdx);
+ maxResultBufIdx = appendTupleToNewResults(btreeCursor, maxResultBufIdx);
}
btreeCursor.reset();
} catch (Exception e) {
@@ -194,7 +194,7 @@
}
private int intersectList(IBTreeCursor btreeCursor, List<ByteBuffer> prevResultBuffers, int maxPrevBufIdx, List<ByteBuffer> newResultBuffers) throws IOException, Exception {
-
+
int newBufIdx = 0;
ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
diff --git a/hyracks/hyracks-storage-am-invertedindex/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchers/SimpleConjunctiveSearcherTest.java b/hyracks/hyracks-storage-am-invertedindex/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchers/SimpleConjunctiveSearcherTest.java
index fa5ec4c..788e01b 100644
--- a/hyracks/hyracks-storage-am-invertedindex/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchers/SimpleConjunctiveSearcherTest.java
+++ b/hyracks/hyracks-storage-am-invertedindex/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchers/SimpleConjunctiveSearcherTest.java
@@ -35,9 +35,9 @@
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.FieldPrefixNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.MetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.NSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOp;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
@@ -110,8 +110,8 @@
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
//SimpleTupleWriterFactory tupleWriterFactory = new SimpleTupleWriterFactory();
- IBTreeLeafFrameFactory leafFrameFactory = new NSMLeafFrameFactory(tupleWriterFactory);
- //IBTreeLeafFrameFactory leafFrameFactory = new FieldPrefixNSMLeafFrameFactory(tupleWriterFactory);
+ //IBTreeLeafFrameFactory leafFrameFactory = new NSMLeafFrameFactory(tupleWriterFactory);
+ IBTreeLeafFrameFactory leafFrameFactory = new FieldPrefixNSMLeafFrameFactory(tupleWriterFactory);
IBTreeInteriorFrameFactory interiorFrameFactory = new NSMInteriorFrameFactory(tupleWriterFactory);
IBTreeMetaDataFrameFactory metaFrameFactory = new MetaDataFrameFactory();