Merge branch 'ceej/tiered-tests' - consolidated surefire configuration,
and added ability to enable/disable groupings of tests at the top level.
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java
index aa47041..961e527 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java
@@ -44,7 +44,7 @@
}
public JobSpecification compilePlan(ILogicalPlan plan, IOperatorSchema outerPlanSchema, IJobletEventListenerFactory jobEventListenerFactory) throws AlgebricksException {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(context.getFrameSize());
if (jobEventListenerFactory != null) {
spec.setJobletEventListenerFactory(jobEventListenerFactory);
}
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
index b61c245..e7b469a 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
@@ -32,7 +32,6 @@
public abstract class AbstractRuleController {
protected IOptimizationContext context;
- private LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor();
public AbstractRuleController() {
}
@@ -65,6 +64,7 @@
private String getPlanString(Mutable<ILogicalOperator> opRef) throws AlgebricksException {
if (AlgebricksConfig.ALGEBRICKS_LOGGER.isLoggable(Level.FINE)) {
StringBuilder sb = new StringBuilder();
+ LogicalOperatorPrettyPrintVisitor pvisitor = context.getPrettyPrintVisitor();
PlanPrettyPrinter.printOperator((AbstractLogicalOperator) opRef.getValue(), sb, pvisitor, 0);
return sb.toString();
}
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
index 9035ca5..17c6900 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
@@ -26,7 +26,6 @@
import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
diff --git a/algebricks/algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java b/algebricks/algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
index 974dc4d..2b29448 100644
--- a/algebricks/algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
+++ b/algebricks/algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
@@ -101,6 +101,7 @@
private static final String PATH_ACTUAL = "rttest";
private static final String PATH_BASE = "src" + SEPARATOR + "test" + SEPARATOR + "resources";
private static final String PATH_EXPECTED = PATH_BASE + SEPARATOR + "results";
+ private static final int FRAME_SIZE = 32768;
private static final String[] DEFAULT_NODES = new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID };
@@ -123,7 +124,7 @@
@Test
public void etsAssignPrint() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(FRAME_SIZE);
IntegerConstantEvalFactory const1 = new IntegerConstantEvalFactory(400);
IntegerConstantEvalFactory const2 = new IntegerConstantEvalFactory(3);
@@ -147,7 +148,7 @@
@Test
public void etsAssignWrite() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(FRAME_SIZE);
IntegerConstantEvalFactory const1 = new IntegerConstantEvalFactory(400);
IntegerConstantEvalFactory const2 = new IntegerConstantEvalFactory(3);
@@ -178,7 +179,7 @@
@Test
public void scanSelectWrite() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(FRAME_SIZE);
// the scanner
FileSplit[] intFileSplits = new FileSplit[1];
@@ -224,7 +225,7 @@
@Test
public void etsAssignProjectWrite() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(FRAME_SIZE);
IntegerConstantEvalFactory const1 = new IntegerConstantEvalFactory(400);
IntegerConstantEvalFactory const2 = new IntegerConstantEvalFactory(3);
@@ -261,7 +262,7 @@
@Test
public void scanLimitWrite() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(FRAME_SIZE);
// the scanner
FileSplit[] fileSplits = new FileSplit[1];
@@ -312,7 +313,7 @@
@Test
public void etsUnnestWrite() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(FRAME_SIZE);
EmptyTupleSourceRuntimeFactory ets = new EmptyTupleSourceRuntimeFactory();
RecordDescriptor etsDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
@@ -342,7 +343,7 @@
@Test
public void scanAggregateWrite() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(FRAME_SIZE);
// the scanner
FileSplit[] fileSplits = new FileSplit[1];
@@ -394,7 +395,7 @@
@Test
public void scanSortGbySelectWrite() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(FRAME_SIZE);
// the scanner
FileSplit[] fileSplits = new FileSplit[1];
@@ -479,7 +480,7 @@
@Test
public void scanHashGbySelectWrite() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(FRAME_SIZE);
// the scanner
FileSplit[] fileSplits = new FileSplit[1];
@@ -548,7 +549,7 @@
@Test
public void etsUnnestRunningaggregateWrite() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(FRAME_SIZE);
EmptyTupleSourceRuntimeFactory ets = new EmptyTupleSourceRuntimeFactory();
RecordDescriptor etsDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
@@ -587,7 +588,7 @@
@Test
public void etsAssignScriptWrite() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(FRAME_SIZE);
IntegerConstantEvalFactory const1 = new IntegerConstantEvalFactory(400);
IntegerConstantEvalFactory const2 = new IntegerConstantEvalFactory(3);
@@ -643,7 +644,7 @@
public void scanSplitWrite() throws Exception {
final int outputArity = 2;
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(FRAME_SIZE);
String inputFileName = "data/tpch0.001/customer.tbl";
File inputFile = new File(inputFileName);
@@ -696,7 +697,7 @@
@Test
public void scanMicroSortWrite() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(FRAME_SIZE);
// the scanner
FileSplit[] fileSplits = new FileSplit[1];
@@ -744,7 +745,7 @@
@Test
public void etsAssignSubplanProjectWrite() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(FRAME_SIZE);
IntegerConstantEvalFactory const1 = new IntegerConstantEvalFactory(400);
IntegerConstantEvalFactory const2 = new IntegerConstantEvalFactory(3);
@@ -804,7 +805,7 @@
@Test
public void scanMicroSortGbySelectWrite() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(FRAME_SIZE);
// the scanner
FileSplit[] fileSplits = new FileSplit[1];
diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java
index 0873582..f0e8510 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java
@@ -38,6 +38,8 @@
public class JobSpecification implements Serializable, IOperatorDescriptorRegistry, IConnectorDescriptorRegistry {
private static final long serialVersionUID = 1L;
+
+ private static final int DEFAULT_FRAME_SIZE = 32768;
private final List<OperatorDescriptorId> roots;
@@ -73,7 +75,13 @@
private transient int connectorIdCounter;
+ // This constructor uses the default frame size. It is for test purposes only.
+ // For other use cases, use the one which sets the frame size.
public JobSpecification() {
+ this(DEFAULT_FRAME_SIZE);
+ }
+
+ public JobSpecification(int frameSize) {
roots = new ArrayList<OperatorDescriptorId>();
resultSetIds = new ArrayList<ResultSetId>();
opMap = new HashMap<OperatorDescriptorId, IOperatorDescriptor>();
@@ -85,9 +93,9 @@
userConstraints = new HashSet<Constraint>();
operatorIdCounter = 0;
connectorIdCounter = 0;
- frameSize = 32768;
maxReattempts = 2;
useConnectorPolicyForScheduling = true;
+ setFrameSize(frameSize);
}
@Override
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/NestedLoopJoin.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/NestedLoopJoin.java
index 994b0ef..52f1198 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/NestedLoopJoin.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/NestedLoopJoin.java
@@ -100,14 +100,14 @@
reloadFrame(outerBuffer);
return;
}
- for (ByteBuffer outBuffer : outBuffers) {
- runFileReader = runFileWriter.createReader();
- runFileReader.open();
- while (runFileReader.nextFrame(innerBuffer)) {
+ runFileReader = runFileWriter.createReader();
+ runFileReader.open();
+ while (runFileReader.nextFrame(innerBuffer)) {
+ for (ByteBuffer outBuffer : outBuffers) {
blockJoin(outBuffer, innerBuffer, writer);
}
- runFileReader.close();
}
+ runFileReader.close();
currentMemSize = 0;
reloadFrame(outerBuffer);
}
@@ -144,14 +144,14 @@
}
if (!matchFound && isLeftOuter) {
- if (!appender.appendConcat(accessorOuter, i, nullTupleBuilder.getFieldEndOffsets(),
- nullTupleBuilder.getByteArray(), 0, nullTupleBuilder.getSize())) {
+ final int[] ntFieldEndOffsets = nullTupleBuilder.getFieldEndOffsets();
+ final byte[] ntByteArray = nullTupleBuilder.getByteArray();
+ final int ntSize = nullTupleBuilder.getSize();
+ if (!appender.appendConcat(accessorOuter, i, ntFieldEndOffsets, ntByteArray, 0, ntSize)) {
flushFrame(outBuffer, writer);
appender.reset(outBuffer, true);
- if (!appender.appendConcat(accessorOuter, i, nullTupleBuilder.getFieldEndOffsets(),
- nullTupleBuilder.getByteArray(), 0, nullTupleBuilder.getSize())) {
- int tSize = accessorOuter.getTupleEndOffset(i) - accessorOuter.getTupleStartOffset(i)
- + nullTupleBuilder.getSize();
+ if (!appender.appendConcat(accessorOuter, i, ntFieldEndOffsets, ntByteArray, 0, ntSize)) {
+ int tSize = accessorOuter.getTupleEndOffset(i) - accessorOuter.getTupleStartOffset(i) + ntSize;
throw new HyracksDataException("Record size (" + tSize + ") larger than frame size ("
+ appender.getBuffer().capacity() + ")");
}
@@ -169,32 +169,27 @@
}
}
- private void appendToResults(int outerTupleId, int innerTupleId, IFrameWriter writer) throws HyracksDataException{
- if(!isReversed){
- if (!appender.appendConcat(accessorOuter, outerTupleId, accessorInner, innerTupleId)) {
- flushFrame(outBuffer, writer);
- appender.reset(outBuffer, true);
- if (!appender.appendConcat(accessorOuter, outerTupleId, accessorInner, innerTupleId)) {
- int tSize = accessorOuter.getTupleEndOffset(outerTupleId) - accessorOuter.getTupleStartOffset(outerTupleId)
- + accessorInner.getTupleEndOffset(innerTupleId) - accessorInner.getTupleStartOffset(innerTupleId);
- throw new HyracksDataException("Record size (" + tSize + ") larger than frame size ("
- + appender.getBuffer().capacity() + ")");
- }
+ private void appendToResults(int outerTupleId, int innerTupleId, IFrameWriter writer) throws HyracksDataException {
+ if (!isReversed) {
+ appendResultToFrame(accessorOuter, outerTupleId, accessorInner, innerTupleId, writer);
+ } else {
+ //Role Reversal Optimization is triggered
+ appendResultToFrame(accessorInner, innerTupleId, accessorOuter, outerTupleId, writer);
+ }
+ }
+
+ private void appendResultToFrame(FrameTupleAccessor accessor1, int tupleId1, FrameTupleAccessor accessor2,
+ int tupleId2, IFrameWriter writer) throws HyracksDataException {
+ if (!appender.appendConcat(accessor1, tupleId1, accessor2, tupleId2)) {
+ flushFrame(outBuffer, writer);
+ appender.reset(outBuffer, true);
+ if (!appender.appendConcat(accessor1, tupleId1, accessor2, tupleId2)) {
+ int tSize = accessor1.getTupleEndOffset(tupleId1) - accessor1.getTupleStartOffset(tupleId1)
+ + accessor2.getTupleEndOffset(tupleId2) - accessor2.getTupleStartOffset(tupleId2);
+ throw new HyracksDataException("Record size (" + tSize + ") larger than frame size ("
+ + appender.getBuffer().capacity() + ")");
}
- }
- else{ //Role Reversal Optimization is triggered
- if (!appender.appendConcat(accessorInner, innerTupleId, accessorOuter, outerTupleId)) {
- flushFrame(outBuffer, writer);
- appender.reset(outBuffer, true);
- if (!appender.appendConcat(accessorInner, innerTupleId, accessorOuter, outerTupleId)) {
- int tSize = accessorInner.getTupleEndOffset(innerTupleId) - accessorInner.getTupleStartOffset(innerTupleId)+
- accessorOuter.getTupleEndOffset(outerTupleId) - accessorOuter.getTupleStartOffset(outerTupleId);
- throw new HyracksDataException("Record size (" + tSize + ") larger than frame size ("
- + appender.getBuffer().capacity() + ")");
- }
- }
- }
-
+ }
}
public void closeCache() throws HyracksDataException {
@@ -204,15 +199,14 @@
}
public void closeJoin(IFrameWriter writer) throws HyracksDataException {
- for (int i = 0; i < currentMemSize; i++) {
- ByteBuffer outBuffer = outBuffers.get(i);
- runFileReader = runFileWriter.createReader();
- runFileReader.open();
- while (runFileReader.nextFrame(innerBuffer)) {
- blockJoin(outBuffer, innerBuffer, writer);
+ runFileReader = runFileWriter.createReader();
+ runFileReader.open();
+ while (runFileReader.nextFrame(innerBuffer)) {
+ for (int i = 0; i < currentMemSize; i++) {
+ blockJoin(outBuffers.get(i), innerBuffer, writer);
}
- runFileReader.close();
}
+ runFileReader.close();
outBuffers.clear();
currentMemSize = 0;
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
index cde561e..a18d496 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
@@ -53,7 +53,7 @@
// This example will insert tuples into the primary and secondary index using an insert pipeline
-public class InsertPipelineExample {
+public class InsertPipelineExample {
private static class Options {
@Option(name = "-host", usage = "Hyracks Cluster Controller Host name", required = true)
public String host;
@@ -72,6 +72,9 @@
@Option(name = "-secondary-btreename", usage = "B-Tree file name of secondary index", required = true)
public String secondaryBTreeName;
+
+ @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
+ public int frameSize = 32768;
}
public static void main(String[] args) throws Exception {
@@ -92,7 +95,7 @@
private static JobSpecification createJob(Options options) {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(options.frameSize);
String[] splitNCs = options.ncs.split(",");
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
index a027570..1c1901d 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
@@ -51,7 +51,7 @@
// This example will load a primary index from randomly generated data
-public class PrimaryIndexBulkLoadExample {
+public class PrimaryIndexBulkLoadExample {
private static class Options {
@Option(name = "-host", usage = "Hyracks Cluster Controller Host name", required = true)
public String host;
@@ -70,6 +70,9 @@
@Option(name = "-sortbuffer-size", usage = "Sort buffer size in frames (default: 32768)", required = false)
public int sbSize = 32768;
+
+ @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
+ public int frameSize = 32768;
}
public static void main(String[] args) throws Exception {
@@ -90,7 +93,7 @@
private static JobSpecification createJob(Options options) {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(options.frameSize);
String[] splitNCs = options.ncs.split(",");
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
index a36fbdd..01004e6f 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
@@ -63,6 +63,9 @@
@Option(name = "-btreename", usage = "B-Tree file name to search", required = true)
public String btreeName;
+
+ @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
+ public int frameSize = 32768;
}
public static void main(String[] args) throws Exception {
@@ -83,7 +86,7 @@
private static JobSpecification createJob(Options options) throws HyracksDataException {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(options.frameSize);
String[] splitNCs = options.ncs.split(",");
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
index 805660f..4826808 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
@@ -65,6 +65,9 @@
@Option(name = "-sortbuffer-size", usage = "Sort buffer size in frames (default: 32768)", required = false)
public int sbSize = 32768;
+
+ @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
+ public int frameSize = 32768;
}
public static void main(String[] args) throws Exception {
@@ -85,7 +88,7 @@
private static JobSpecification createJob(Options options) {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(options.frameSize);
String[] splitNCs = options.ncs.split(",");
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
index fc5a37a..5ff5003 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
@@ -66,6 +66,9 @@
@Option(name = "-secondary-btreename", usage = "Secondary B-Tree file name to search", required = true)
public String secondaryBTreeName;
+
+ @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
+ public int frameSize = 32768;
}
public static void main(String[] args) throws Exception {
@@ -86,7 +89,7 @@
private static JobSpecification createJob(Options options) throws HyracksDataException {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(options.frameSize);
String[] splitNCs = options.ncs.split(",");
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060025932/device_id_0/2014-03-13-18-00-26-469_2014-03-13-18-00-26-469_b b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060025932/device_id_0/2014-03-13-18-00-26-469_2014-03-13-18-00-26-469_b
new file mode 100644
index 0000000..e929192
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060025932/device_id_0/2014-03-13-18-00-26-469_2014-03-13-18-00-26-469_b
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060025932/device_id_0/2014-03-13-18-00-26-469_2014-03-13-18-00-26-469_d b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060025932/device_id_0/2014-03-13-18-00-26-469_2014-03-13-18-00-26-469_d
new file mode 100644
index 0000000..86b254e
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060025932/device_id_0/2014-03-13-18-00-26-469_2014-03-13-18-00-26-469_d
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060025932/device_id_0/2014-03-13-18-00-26-469_2014-03-13-18-00-26-469_f b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060025932/device_id_0/2014-03-13-18-00-26-469_2014-03-13-18-00-26-469_f
new file mode 100644
index 0000000..6d17cf9
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060025932/device_id_0/2014-03-13-18-00-26-469_2014-03-13-18-00-26-469_f
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060025932/device_id_0/2014-03-13-18-00-26-469_2014-03-13-18-00-26-469_i b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060025932/device_id_0/2014-03-13-18-00-26-469_2014-03-13-18-00-26-469_i
new file mode 100644
index 0000000..32a748d
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060025932/device_id_0/2014-03-13-18-00-26-469_2014-03-13-18-00-26-469_i
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060029121/device_id_0/2014-03-13-18-00-29-657_2014-03-13-18-00-29-657_b b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060029121/device_id_0/2014-03-13-18-00-29-657_2014-03-13-18-00-29-657_b
new file mode 100644
index 0000000..62f7c44
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060029121/device_id_0/2014-03-13-18-00-29-657_2014-03-13-18-00-29-657_b
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060029121/device_id_0/2014-03-13-18-00-29-657_2014-03-13-18-00-29-657_d b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060029121/device_id_0/2014-03-13-18-00-29-657_2014-03-13-18-00-29-657_d
new file mode 100644
index 0000000..86b254e
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060029121/device_id_0/2014-03-13-18-00-29-657_2014-03-13-18-00-29-657_d
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060029121/device_id_0/2014-03-13-18-00-29-657_2014-03-13-18-00-29-657_f b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060029121/device_id_0/2014-03-13-18-00-29-657_2014-03-13-18-00-29-657_f
new file mode 100644
index 0000000..6d17cf9
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060029121/device_id_0/2014-03-13-18-00-29-657_2014-03-13-18-00-29-657_f
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060029121/device_id_0/2014-03-13-18-00-29-657_2014-03-13-18-00-29-657_i b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060029121/device_id_0/2014-03-13-18-00-29-657_2014-03-13-18-00-29-657_i
new file mode 100644
index 0000000..409bf7a
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-060029121/device_id_0/2014-03-13-18-00-29-657_2014-03-13-18-00-29-657_i
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070436429/device_id_0/2014-03-13-19-04-36-989_2014-03-13-19-04-36-989_b b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070436429/device_id_0/2014-03-13-19-04-36-989_2014-03-13-19-04-36-989_b
new file mode 100644
index 0000000..e929192
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070436429/device_id_0/2014-03-13-19-04-36-989_2014-03-13-19-04-36-989_b
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070436429/device_id_0/2014-03-13-19-04-36-989_2014-03-13-19-04-36-989_d b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070436429/device_id_0/2014-03-13-19-04-36-989_2014-03-13-19-04-36-989_d
new file mode 100644
index 0000000..86b254e
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070436429/device_id_0/2014-03-13-19-04-36-989_2014-03-13-19-04-36-989_d
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070436429/device_id_0/2014-03-13-19-04-36-989_2014-03-13-19-04-36-989_f b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070436429/device_id_0/2014-03-13-19-04-36-989_2014-03-13-19-04-36-989_f
new file mode 100644
index 0000000..6d17cf9
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070436429/device_id_0/2014-03-13-19-04-36-989_2014-03-13-19-04-36-989_f
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070436429/device_id_0/2014-03-13-19-04-36-989_2014-03-13-19-04-36-989_i b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070436429/device_id_0/2014-03-13-19-04-36-989_2014-03-13-19-04-36-989_i
new file mode 100644
index 0000000..32a748d
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070436429/device_id_0/2014-03-13-19-04-36-989_2014-03-13-19-04-36-989_i
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070439620/device_id_0/2014-03-13-19-04-40-195_2014-03-13-19-04-40-195_b b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070439620/device_id_0/2014-03-13-19-04-40-195_2014-03-13-19-04-40-195_b
new file mode 100644
index 0000000..62f7c44
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070439620/device_id_0/2014-03-13-19-04-40-195_2014-03-13-19-04-40-195_b
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070439620/device_id_0/2014-03-13-19-04-40-195_2014-03-13-19-04-40-195_d b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070439620/device_id_0/2014-03-13-19-04-40-195_2014-03-13-19-04-40-195_d
new file mode 100644
index 0000000..86b254e
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070439620/device_id_0/2014-03-13-19-04-40-195_2014-03-13-19-04-40-195_d
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070439620/device_id_0/2014-03-13-19-04-40-195_2014-03-13-19-04-40-195_f b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070439620/device_id_0/2014-03-13-19-04-40-195_2014-03-13-19-04-40-195_f
new file mode 100644
index 0000000..6d17cf9
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070439620/device_id_0/2014-03-13-19-04-40-195_2014-03-13-19-04-40-195_f
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070439620/device_id_0/2014-03-13-19-04-40-195_2014-03-13-19-04-40-195_i b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070439620/device_id_0/2014-03-13-19-04-40-195_2014-03-13-19-04-40-195_i
new file mode 100644
index 0000000..409bf7a
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-070439620/device_id_0/2014-03-13-19-04-40-195_2014-03-13-19-04-40-195_i
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105003450/device_id_0/2014-03-13-22-50-03-981_2014-03-13-22-50-03-981_b b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105003450/device_id_0/2014-03-13-22-50-03-981_2014-03-13-22-50-03-981_b
new file mode 100644
index 0000000..e929192
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105003450/device_id_0/2014-03-13-22-50-03-981_2014-03-13-22-50-03-981_b
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105003450/device_id_0/2014-03-13-22-50-03-981_2014-03-13-22-50-03-981_d b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105003450/device_id_0/2014-03-13-22-50-03-981_2014-03-13-22-50-03-981_d
new file mode 100644
index 0000000..86b254e
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105003450/device_id_0/2014-03-13-22-50-03-981_2014-03-13-22-50-03-981_d
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105003450/device_id_0/2014-03-13-22-50-03-981_2014-03-13-22-50-03-981_f b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105003450/device_id_0/2014-03-13-22-50-03-981_2014-03-13-22-50-03-981_f
new file mode 100644
index 0000000..6d17cf9
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105003450/device_id_0/2014-03-13-22-50-03-981_2014-03-13-22-50-03-981_f
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105003450/device_id_0/2014-03-13-22-50-03-981_2014-03-13-22-50-03-981_i b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105003450/device_id_0/2014-03-13-22-50-03-981_2014-03-13-22-50-03-981_i
new file mode 100644
index 0000000..32a748d
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105003450/device_id_0/2014-03-13-22-50-03-981_2014-03-13-22-50-03-981_i
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105006534/device_id_0/2014-03-13-22-50-07-110_2014-03-13-22-50-07-110_b b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105006534/device_id_0/2014-03-13-22-50-07-110_2014-03-13-22-50-07-110_b
new file mode 100644
index 0000000..62f7c44
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105006534/device_id_0/2014-03-13-22-50-07-110_2014-03-13-22-50-07-110_b
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105006534/device_id_0/2014-03-13-22-50-07-110_2014-03-13-22-50-07-110_d b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105006534/device_id_0/2014-03-13-22-50-07-110_2014-03-13-22-50-07-110_d
new file mode 100644
index 0000000..86b254e
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105006534/device_id_0/2014-03-13-22-50-07-110_2014-03-13-22-50-07-110_d
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105006534/device_id_0/2014-03-13-22-50-07-110_2014-03-13-22-50-07-110_f b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105006534/device_id_0/2014-03-13-22-50-07-110_2014-03-13-22-50-07-110_f
new file mode 100644
index 0000000..6d17cf9
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105006534/device_id_0/2014-03-13-22-50-07-110_2014-03-13-22-50-07-110_f
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105006534/device_id_0/2014-03-13-22-50-07-110_2014-03-13-22-50-07-110_i b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105006534/device_id_0/2014-03-13-22-50-07-110_2014-03-13-22-50-07-110_i
new file mode 100644
index 0000000..409bf7a
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/invIndexBtree130314-105006534/device_id_0/2014-03-13-22-50-07-110_2014-03-13-22-50-07-110_i
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-060025932/device_id_0 b/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-060025932/device_id_0
new file mode 100644
index 0000000..78654fc
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-060025932/device_id_0
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-060029121/device_id_0 b/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-060029121/device_id_0
new file mode 100644
index 0000000..78654fc
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-060029121/device_id_0
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-070436429/device_id_0 b/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-070436429/device_id_0
new file mode 100644
index 0000000..78654fc
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-070436429/device_id_0
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-070439620/device_id_0 b/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-070439620/device_id_0
new file mode 100644
index 0000000..78654fc
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-070439620/device_id_0
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-105003450/device_id_0 b/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-105003450/device_id_0
new file mode 100644
index 0000000..78654fc
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-105003450/device_id_0
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-105006534/device_id_0 b/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-105006534/device_id_0
new file mode 100644
index 0000000..78654fc
--- /dev/null
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/primaryBtree130314-105006534/device_id_0
Binary files differ
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java
index a4fd4c1..f4d422b 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java
@@ -43,7 +43,7 @@
return new LSMBTreeDataflowHelperFactory(virtualBufferCacheProvider, new ConstantMergePolicyFactory(),
MERGE_POLICY_PROPERTIES, ThreadCountingOperationTrackerProvider.INSTANCE,
SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE,
- DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE);
+ DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, true);
}
}
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
index 79b8eb7..e848c7d 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
@@ -45,10 +45,12 @@
public IIndexDataflowHelperFactory createDataFlowHelperFactory(
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
- IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory) {
+ IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory,
+ int[] btreeFields) {
return new LSMRTreeDataflowHelperFactory(valueProviderFactories, rtreePolicyType, btreeComparatorFactories,
virtualBufferCacheProvider, new ConstantMergePolicyFactory(), MERGE_POLICY_PROPERTIES,
ThreadCountingOperationTrackerProvider.INSTANCE, SynchronousSchedulerProvider.INSTANCE,
- NoOpIOOperationCallback.INSTANCE, linearizerCmpFactory, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE);
+ NoOpIOOperationCallback.INSTANCE, linearizerCmpFactory, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE,
+ btreeFields);
}
}
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeSecondaryIndexInsertOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeSecondaryIndexInsertOperatorTest.java
index ab6ffdd..8e53269 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeSecondaryIndexInsertOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeSecondaryIndexInsertOperatorTest.java
@@ -26,6 +26,11 @@
import edu.uci.ics.hyracks.tests.am.rtree.RTreeSecondaryIndexInsertOperatorTest;
public class LSMRTreeSecondaryIndexInsertOperatorTest extends RTreeSecondaryIndexInsertOperatorTest {
+
+ public LSMRTreeSecondaryIndexInsertOperatorTest() {
+ this.rTreeType = RTreeType.LSMRTREE;
+ }
+
protected ITreeIndexOperatorTestHelper createTestHelper() throws HyracksException {
return new LSMRTreeOperatorTestHelper(TestStorageManagerComponentHolder.getIOManager());
}
@@ -33,8 +38,9 @@
@Override
protected IIndexDataflowHelperFactory createDataFlowHelperFactory(
IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType,
- IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory) {
+ IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory,
+ int[] btreeFields) {
return ((LSMRTreeOperatorTestHelper) testHelper).createDataFlowHelperFactory(secondaryValueProviderFactories,
- rtreePolicyType, btreeComparatorFactories, linearizerCmpFactory);
+ rtreePolicyType, btreeComparatorFactories, linearizerCmpFactory, btreeFields);
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeSecondaryIndexSearchOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeSecondaryIndexSearchOperatorTest.java
index 591a75d..67691ea 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeSecondaryIndexSearchOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeSecondaryIndexSearchOperatorTest.java
@@ -26,6 +26,10 @@
import edu.uci.ics.hyracks.tests.am.rtree.RTreeSecondaryIndexSearchOperatorTest;
public class LSMRTreeSecondaryIndexSearchOperatorTest extends RTreeSecondaryIndexSearchOperatorTest {
+ public LSMRTreeSecondaryIndexSearchOperatorTest() {
+ this.rTreeType = RTreeType.LSMRTREE;
+ }
+
protected ITreeIndexOperatorTestHelper createTestHelper() throws HyracksException {
return new LSMRTreeOperatorTestHelper(TestStorageManagerComponentHolder.getIOManager());
}
@@ -33,8 +37,9 @@
@Override
protected IIndexDataflowHelperFactory createDataFlowHelperFactory(
IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType,
- IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory) {
+ IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory,
+ int[] btreeFields) {
return ((LSMRTreeOperatorTestHelper) testHelper).createDataFlowHelperFactory(secondaryValueProviderFactories,
- rtreePolicyType, btreeComparatorFactories, linearizerCmpFactory);
+ rtreePolicyType, btreeComparatorFactories, linearizerCmpFactory, btreeFields);
}
}
\ No newline at end of file
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesSecondaryIndexInsertOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesSecondaryIndexInsertOperatorTest.java
index 29816e3..ddd6e10 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesSecondaryIndexInsertOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesSecondaryIndexInsertOperatorTest.java
@@ -26,6 +26,10 @@
import edu.uci.ics.hyracks.tests.am.rtree.RTreeSecondaryIndexInsertOperatorTest;
public class LSMRTreeWithAntiMatterTuplesSecondaryIndexInsertOperatorTest extends RTreeSecondaryIndexInsertOperatorTest {
+ public LSMRTreeWithAntiMatterTuplesSecondaryIndexInsertOperatorTest() {
+ this.rTreeType = RTreeType.LSMRTREE_WITH_ANTIMATTER;
+ }
+
protected ITreeIndexOperatorTestHelper createTestHelper() throws HyracksException {
return new LSMRTreeWithAntiMatterTuplesOperatorTestHelper(TestStorageManagerComponentHolder.getIOManager());
}
@@ -33,7 +37,8 @@
@Override
protected IIndexDataflowHelperFactory createDataFlowHelperFactory(
IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType,
- IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory) {
+ IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory,
+ int[] btreeFields) {
return ((LSMRTreeWithAntiMatterTuplesOperatorTestHelper) testHelper).createDataFlowHelperFactory(
secondaryValueProviderFactories, rtreePolicyType, btreeComparatorFactories, linearizerCmpFactory);
}
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesSecondaryIndexSearchOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesSecondaryIndexSearchOperatorTest.java
index 2cf1418..f4fad79 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesSecondaryIndexSearchOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesSecondaryIndexSearchOperatorTest.java
@@ -26,6 +26,10 @@
import edu.uci.ics.hyracks.tests.am.rtree.RTreeSecondaryIndexSearchOperatorTest;
public class LSMRTreeWithAntiMatterTuplesSecondaryIndexSearchOperatorTest extends RTreeSecondaryIndexSearchOperatorTest {
+ public LSMRTreeWithAntiMatterTuplesSecondaryIndexSearchOperatorTest() {
+ this.rTreeType = RTreeType.LSMRTREE_WITH_ANTIMATTER;
+ }
+
protected ITreeIndexOperatorTestHelper createTestHelper() throws HyracksException {
return new LSMRTreeWithAntiMatterTuplesOperatorTestHelper(TestStorageManagerComponentHolder.getIOManager());
}
@@ -33,7 +37,8 @@
@Override
protected IIndexDataflowHelperFactory createDataFlowHelperFactory(
IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType,
- IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory) {
+ IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory,
+ int[] btreeFields) {
return ((LSMRTreeWithAntiMatterTuplesOperatorTestHelper) testHelper).createDataFlowHelperFactory(
secondaryValueProviderFactories, rtreePolicyType, btreeComparatorFactories, linearizerCmpFactory);
}
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
index d1c682c..ee08868 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
@@ -76,6 +76,14 @@
TestStorageManagerComponentHolder.init(8192, 20, 20);
}
+ protected enum RTreeType {
+ LSMRTREE,
+ LSMRTREE_WITH_ANTIMATTER,
+ RTREE
+ };
+
+ protected RTreeType rTreeType;
+
protected final IStorageManagerInterface storageManager = new TestStorageManagerInterface();
protected final IIndexLifecycleManagerProvider lcManagerProvider = new TestIndexLifecycleManagerProvider();
protected IIndexDataflowHelperFactory rtreeDataflowHelperFactory;
@@ -111,8 +119,8 @@
// This is only used for the LSMRTree. We need a comparator Factories for
// the BTree component of the LSMRTree.
- protected final int btreeKeyFieldCount = 5;
- protected final IBinaryComparatorFactory[] btreeComparatorFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ protected int btreeKeyFieldCount = 5;
+ protected IBinaryComparatorFactory[] btreeComparatorFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
protected String secondaryFileName;
protected IFileSplitProvider secondarySplitProvider;
@@ -159,24 +167,35 @@
secondaryComparatorFactories[3] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
// This only used for LSMRTree
- btreeComparatorFactories[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
- btreeComparatorFactories[1] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
- btreeComparatorFactories[2] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
- btreeComparatorFactories[3] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
- btreeComparatorFactories[4] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ int[] btreeFields = null;
+ if (rTreeType == RTreeType.LSMRTREE) {
+ btreeKeyFieldCount = 1;
+ btreeComparatorFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ btreeFields = new int[1];
+ btreeFields[0] = 4;
+
+ } else {
+ btreeComparatorFactories[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeComparatorFactories[1] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeComparatorFactories[2] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeComparatorFactories[3] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeComparatorFactories[4] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ }
IPrimitiveValueProviderFactory[] secondaryValueProviderFactories = RTreeUtils
.createPrimitiveValueProviderFactories(secondaryComparatorFactories.length, DoublePointable.FACTORY);
rtreeDataflowHelperFactory = createDataFlowHelperFactory(secondaryValueProviderFactories,
RTreePolicyType.RSTARTREE, btreeComparatorFactories,
- LSMRTreeUtils.proposeBestLinearizer(secondaryTypeTraits, secondaryComparatorFactories.length));
+ LSMRTreeUtils.proposeBestLinearizer(secondaryTypeTraits, secondaryComparatorFactories.length), btreeFields);
}
protected abstract IIndexDataflowHelperFactory createDataFlowHelperFactory(
IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType,
- IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory)
+ IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory,
+ int[] btreeFields)
throws TreeIndexException;
protected void createPrimaryIndex() throws Exception {
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java
index a888538..32380de 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java
@@ -45,6 +45,10 @@
public class RTreeSecondaryIndexInsertOperatorTest extends AbstractRTreeOperatorTest {
+ public RTreeSecondaryIndexInsertOperatorTest() {
+ this.rTreeType = RTreeType.RTREE;
+ }
+
@Before
public void setup() throws Exception {
super.setup();
@@ -118,8 +122,8 @@
@Override
protected IIndexDataflowHelperFactory createDataFlowHelperFactory(
IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType,
- IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory)
- throws TreeIndexException {
+ IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory,
+ int[] btreeFields) throws TreeIndexException {
return ((RTreeOperatorTestHelper) testHelper).createDataFlowHelperFactory(secondaryValueProviderFactories,
rtreePolicyType, null);
}
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java
index 9ec0f7a..40d9e5b 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java
@@ -43,6 +43,10 @@
public class RTreeSecondaryIndexScanOperatorTest extends AbstractRTreeOperatorTest {
+ public RTreeSecondaryIndexScanOperatorTest() {
+ this.rTreeType = RTreeType.RTREE;
+ }
+
@Before
public void setup() throws Exception {
super.setup();
@@ -102,11 +106,12 @@
@Override
protected IIndexDataflowHelperFactory createDataFlowHelperFactory(
IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType,
- IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory) {
+ IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory,
+ int[] btreeFields) {
return ((RTreeOperatorTestHelper) testHelper).createDataFlowHelperFactory(secondaryValueProviderFactories,
rtreePolicyType, null);
}
-
+
@Override
public void cleanup() throws Exception {
destroyPrimaryIndex();
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java
index 9613920d..b6f1000 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java
@@ -44,6 +44,9 @@
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
public class RTreeSecondaryIndexSearchOperatorTest extends AbstractRTreeOperatorTest {
+ public RTreeSecondaryIndexSearchOperatorTest() {
+ this.rTreeType = RTreeType.RTREE;
+ }
@Before
public void setup() throws Exception {
@@ -117,8 +120,8 @@
@Override
protected IIndexDataflowHelperFactory createDataFlowHelperFactory(
IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType,
- IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory)
- throws TreeIndexException {
+ IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory,
+ int[] btreeFields) throws TreeIndexException {
return ((RTreeOperatorTestHelper) testHelper).createDataFlowHelperFactory(secondaryValueProviderFactories,
rtreePolicyType, null);
}
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java
index ab48d39..438e17d 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java
@@ -35,6 +35,9 @@
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
public class RTreeSecondaryIndexStatsOperatorTest extends AbstractRTreeOperatorTest {
+ public RTreeSecondaryIndexStatsOperatorTest() {
+ this.rTreeType = RTreeType.RTREE;
+ }
@Before
public void setup() throws Exception {
@@ -66,7 +69,8 @@
@Override
protected IIndexDataflowHelperFactory createDataFlowHelperFactory(
IPrimitiveValueProviderFactory[] secondaryValueProviderFactories, RTreePolicyType rtreePolicyType,
- IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory) {
+ IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory,
+ int[] btreeFields) {
return ((RTreeOperatorTestHelper) testHelper).createDataFlowHelperFactory(secondaryValueProviderFactories,
rtreePolicyType, null);
}
diff --git a/hyracks/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/ExternalGroupClient.java b/hyracks/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/ExternalGroupClient.java
index be8e986..1629a1e 100644
--- a/hyracks/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/ExternalGroupClient.java
+++ b/hyracks/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/ExternalGroupClient.java
@@ -83,8 +83,8 @@
@Option(name = "-hashtable-size", usage = "Hash table size (default: 8191)", required = false)
public int htSize = 8191;
- @Option(name = "-frames-limit", usage = "Frame size (default: 32768)", required = false)
- public int framesLimit = 32768;
+ @Option(name = "-frame-size", usage = "Frame size (default: 32768)", required = false)
+ public int frameSize = 32768;
@Option(name = "-sortbuffer-size", usage = "Sort buffer size in frames (default: 512)", required = false)
public int sbSize = 512;
@@ -114,7 +114,7 @@
for (int i = 0; i < 6; i++) {
long start = System.currentTimeMillis();
job = createJob(parseFileSplits(options.inFileSplits), parseFileSplits(options.outFileSplits, i),
- options.htSize, options.sbSize, options.framesLimit, options.sortOutput, options.algo,
+ options.htSize, options.sbSize, options.frameSize, options.sortOutput, options.algo,
options.outPlain);
System.out.print(i + "\t" + (System.currentTimeMillis() - start));
@@ -155,8 +155,8 @@
}
private static JobSpecification createJob(FileSplit[] inSplits, FileSplit[] outSplits, int htSize, int sbSize,
- int framesLimit, boolean sortOutput, int alg, boolean outPlain) {
- JobSpecification spec = new JobSpecification();
+ int frameSize, boolean sortOutput, int alg, boolean outPlain) {
+ JobSpecification spec = new JobSpecification(frameSize);
IFileSplitProvider splitsProvider = new ConstantFileSplitProvider(inSplits);
RecordDescriptor inDesc = new RecordDescriptor(new ISerializerDeserializer[] {
@@ -198,7 +198,7 @@
switch (alg) {
case 0: // new external hash graph
grouper = new edu.uci.ics.hyracks.dataflow.std.group.external.ExternalGroupOperatorDescriptor(spec,
- keys, framesLimit, new IBinaryComparatorFactory[] {
+ keys, frameSize, new IBinaryComparatorFactory[] {
// PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY),
PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) },
new IntegerNormalizedKeyComputerFactory(), new MultiFieldsAggregatorFactory(
@@ -221,7 +221,7 @@
break;
case 1: // External-sort + new-precluster
- ExternalSortOperatorDescriptor sorter2 = new ExternalSortOperatorDescriptor(spec, framesLimit, keys,
+ ExternalSortOperatorDescriptor sorter2 = new ExternalSortOperatorDescriptor(spec, frameSize, keys,
new IBinaryComparatorFactory[] {
// PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY),
PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) }, inDesc);
@@ -271,7 +271,7 @@
break;
default:
grouper = new edu.uci.ics.hyracks.dataflow.std.group.external.ExternalGroupOperatorDescriptor(spec,
- keys, framesLimit, new IBinaryComparatorFactory[] {
+ keys, frameSize, new IBinaryComparatorFactory[] {
// PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY),
PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) },
new IntegerNormalizedKeyComputerFactory(), new MultiFieldsAggregatorFactory(
diff --git a/hyracks/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java b/hyracks/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java
index d342082..82421ae 100644
--- a/hyracks/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java
+++ b/hyracks/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java
@@ -85,6 +85,9 @@
@Option(name = "-runtime-profiling", usage = "Indicates if runtime profiling should be enabled. (default: false)")
public boolean runtimeProfiling = false;
+
+ @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
+ public int frameSize = 32768;
}
public static void main(String[] args) throws Exception {
@@ -95,7 +98,7 @@
IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
JobSpecification job = createJob(parseFileSplits(options.inFileSplits), parseFileSplits(options.outFileSplits),
- options.algo, options.htSize, options.sbSize, options.format);
+ options.algo, options.htSize, options.sbSize, options.format, options.frameSize);
long start = System.currentTimeMillis();
JobId jobId = hcc.startJob(job,
@@ -120,8 +123,8 @@
}
private static JobSpecification createJob(FileSplit[] inSplits, FileSplit[] outSplits, String algo, int htSize,
- int sbSize, String format) {
- JobSpecification spec = new JobSpecification();
+ int sbSize, String format, int frameSize) {
+ JobSpecification spec = new JobSpecification(frameSize);
IFileSplitProvider splitsProvider = new ConstantFileSplitProvider(inSplits);
RecordDescriptor wordDesc = new RecordDescriptor(
diff --git a/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/edu/uci/ics/hyracks/examples/tpch/client/Main.java b/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/edu/uci/ics/hyracks/examples/tpch/client/Main.java
index 6017e8f..a54e940 100644
--- a/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/edu/uci/ics/hyracks/examples/tpch/client/Main.java
+++ b/hyracks/hyracks-examples/tpch-example/tpchclient/src/main/java/edu/uci/ics/hyracks/examples/tpch/client/Main.java
@@ -109,6 +109,9 @@
// Whether group-by is processed after the join
@Option(name = "-has-groupby", usage = "Whether to have group-by operation after join (default: disabled)", required = false)
public boolean hasGroupBy = false;
+
+ @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
+ public int frameSize = 32768;
}
public static void main(String[] args) throws Exception {
@@ -121,7 +124,7 @@
JobSpecification job = createJob(parseFileSplits(options.inFileCustomerSplits),
parseFileSplits(options.inFileOrderSplits), parseFileSplits(options.outFileSplits),
options.numJoinPartitions, options.algo, options.graceInputSize, options.graceRecordsPerFrame,
- options.graceFactor, options.memSize, options.tableSize, options.hasGroupBy);
+ options.graceFactor, options.memSize, options.tableSize, options.hasGroupBy, options.frameSize);
long start = System.currentTimeMillis();
JobId jobId = hcc.startJob(job,
@@ -147,8 +150,8 @@
private static JobSpecification createJob(FileSplit[] customerSplits, FileSplit[] orderSplits,
FileSplit[] resultSplits, int numJoinPartitions, String algo, int graceInputSize, int graceRecordsPerFrame,
- double graceFactor, int memSize, int tableSize, boolean hasGroupBy) throws HyracksDataException {
- JobSpecification spec = new JobSpecification();
+ double graceFactor, int memSize, int tableSize, boolean hasGroupBy, int frameSize) throws HyracksDataException {
+ JobSpecification spec = new JobSpecification(frameSize);
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(customerSplits);
RecordDescriptor custDesc = new RecordDescriptor(new ISerializerDeserializer[] {
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/DualTupleReference.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/DualTupleReference.java
new file mode 100644
index 0000000..0733216
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/DualTupleReference.java
@@ -0,0 +1,42 @@
+package edu.uci.ics.hyracks.storage.am.common.tuples;
+
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+public class DualTupleReference implements ITupleReference {
+
+ private PermutingTupleReference permutingTuple;
+ private ITupleReference tuple;
+
+ public DualTupleReference(int[] fieldPermutation) {
+ permutingTuple = new PermutingTupleReference(fieldPermutation);
+ }
+
+ @Override
+ public int getFieldCount() {
+ return tuple.getFieldCount();
+ }
+
+ @Override
+ public byte[] getFieldData(int fIdx) {
+ return tuple.getFieldData(fIdx);
+ }
+
+ @Override
+ public int getFieldStart(int fIdx) {
+ return tuple.getFieldStart(fIdx);
+ }
+
+ @Override
+ public int getFieldLength(int fIdx) {
+ return tuple.getFieldLength(fIdx);
+ }
+
+ public void reset(ITupleReference tuple) {
+ this.tuple = tuple;
+ permutingTuple.reset(tuple);
+ }
+
+ public ITupleReference getPermutingTuple() {
+ return permutingTuple;
+ }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
index 8a3c313..2708a81 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
@@ -31,21 +31,24 @@
import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.AbstractLSMIndexDataflowHelper;
public class LSMBTreeDataflowHelper extends AbstractLSMIndexDataflowHelper {
+
+ private final boolean needKeyDupCheck;
public LSMBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
List<IVirtualBufferCache> virtualBufferCaches, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck) {
this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
- opTrackerFactory, ioScheduler, ioOpCallbackFactory);
+ opTrackerFactory, ioScheduler, ioOpCallbackFactory, needKeyDupCheck);
}
public LSMBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
List<IVirtualBufferCache> virtualBufferCaches, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory) {
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck) {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackFactory);
+ this.needKeyDupCheck = needKeyDupCheck;
}
@Override
@@ -55,6 +58,6 @@
opDesc.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
treeOpDesc.getTreeIndexComparatorFactories(), treeOpDesc.getTreeIndexBloomFilterKeyFields(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx), ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback());
+ ioOpCallbackFactory.createIOOperationCallback(), needKeyDupCheck);
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
index 0fdef13..7fe7702 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
@@ -30,13 +30,17 @@
public class LSMBTreeDataflowHelperFactory extends AbstractLSMIndexDataflowHelperFactory {
private static final long serialVersionUID = 1L;
+ private final boolean needKeyDupCheck;
public LSMBTreeDataflowHelperFactory(IVirtualBufferCacheProvider virtualBufferCacheProvider,
ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, double bloomFilterFalsePositiveRate) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, double bloomFilterFalsePositiveRate,
+ boolean needKeyDupCheck) {
super(virtualBufferCacheProvider, mergePolicyFactory, mergePolicyProperties, opTrackerFactory,
ioSchedulerProvider, ioOpCallbackFactory, bloomFilterFalsePositiveRate);
+ this.needKeyDupCheck = needKeyDupCheck;
+
}
@Override
@@ -45,6 +49,6 @@
return new LSMBTreeDataflowHelper(opDesc, ctx, partition,
virtualBufferCacheProvider.getVirtualBufferCaches(ctx), bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties), opTrackerFactory,
- ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory);
+ ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, needKeyDupCheck);
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index cb4d268..19307e9 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -85,13 +85,15 @@
private final ITreeIndexFrameFactory deleteLeafFrameFactory;
private final IBinaryComparatorFactory[] cmpFactories;
+ private final boolean needKeyDupCheck;
+
public LSMBTree(List<IVirtualBufferCache> virtualBufferCaches, ITreeIndexFrameFactory interiorFrameFactory,
ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory,
ILSMIndexFileManager fileManager, TreeIndexFactory<BTree> diskBTreeFactory,
TreeIndexFactory<BTree> bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory,
double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount,
IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback) {
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck) {
super(virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback);
int i = 0;
@@ -110,6 +112,7 @@
this.cmpFactories = cmpFactories;
componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory);
bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory);
+ this.needKeyDupCheck = needKeyDupCheck;
}
@Override
@@ -303,43 +306,45 @@
IIndexCursor memCursor = new BTreeRangeSearchCursor(ctx.currentMutableBTreeOpCtx.leafFrame, false);
RangePredicate predicate = new RangePredicate(tuple, tuple, true, true, comparator, comparator);
- // first check the inmemory component
- ctx.currentMutableBTreeAccessor.search(memCursor, predicate);
- try {
- if (memCursor.hasNext()) {
- memCursor.next();
- LSMBTreeTupleReference lsmbtreeTuple = (LSMBTreeTupleReference) memCursor.getTuple();
- if (!lsmbtreeTuple.isAntimatter()) {
- throw new TreeIndexDuplicateKeyException("Failed to insert key since key already exists.");
- } else {
- memCursor.close();
- ctx.currentMutableBTreeAccessor.upsertIfConditionElseInsert(tuple,
- AntimatterAwareTupleAcceptor.INSTANCE);
- return true;
+ if (needKeyDupCheck) {
+ // first check the inmemory component
+ ctx.currentMutableBTreeAccessor.search(memCursor, predicate);
+ try {
+ if (memCursor.hasNext()) {
+ memCursor.next();
+ LSMBTreeTupleReference lsmbtreeTuple = (LSMBTreeTupleReference) memCursor.getTuple();
+ if (!lsmbtreeTuple.isAntimatter()) {
+ throw new TreeIndexDuplicateKeyException("Failed to insert key since key already exists.");
+ } else {
+ memCursor.close();
+ ctx.currentMutableBTreeAccessor.upsertIfConditionElseInsert(tuple,
+ AntimatterAwareTupleAcceptor.INSTANCE);
+ return true;
+ }
}
+ } finally {
+ memCursor.close();
}
- } finally {
- memCursor.close();
- }
- // TODO: Can we just remove the above code that search the mutable
- // component and do it together with the search call below? i.e. instead
- // of passing false to the lsmHarness.search(), we pass true to include
- // the mutable component?
- // the key was not in the inmemory component, so check the disk
- // components
+ // TODO: Can we just remove the above code that search the mutable
+ // component and do it together with the search call below? i.e. instead
+ // of passing false to the lsmHarness.search(), we pass true to include
+ // the mutable component?
+ // the key was not in the inmemory component, so check the disk
+ // components
- // This is a hack to avoid searching the current active mutable component twice. It is critical to add it back once the search is over.
- ILSMComponent firstComponent = ctx.getComponentHolder().remove(0);
- search(ctx, searchCursor, predicate);
- try {
- if (searchCursor.hasNext()) {
- throw new TreeIndexDuplicateKeyException("Failed to insert key since key already exists.");
+ // This is a hack to avoid searching the current active mutable component twice. It is critical to add it back once the search is over.
+ ILSMComponent firstComponent = ctx.getComponentHolder().remove(0);
+ search(ctx, searchCursor, predicate);
+ try {
+ if (searchCursor.hasNext()) {
+ throw new TreeIndexDuplicateKeyException("Failed to insert key since key already exists.");
+ }
+ } finally {
+ searchCursor.close();
+ // Add the current active mutable component back
+ ctx.getComponentHolder().add(0, firstComponent);
}
- } finally {
- searchCursor.close();
- // Add the current active mutable component back
- ctx.getComponentHolder().add(0, firstComponent);
}
ctx.currentMutableBTreeAccessor.upsertIfConditionElseInsert(tuple, AntimatterAwareTupleAcceptor.INSTANCE);
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
index 8b363ff..88148c3 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
@@ -49,7 +49,7 @@
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback) {
+ ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck) {
LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
cmpFactories.length, false);
LSMBTreeTupleWriterFactory deleteTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
@@ -78,7 +78,7 @@
LSMBTree lsmTree = new LSMBTree(virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, cmpFactories, mergePolicy,
- opTracker, ioScheduler, ioOpCallback);
+ opTracker, ioScheduler, ioOpCallback, needKeyDupCheck);
return lsmTree;
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
index 92364fa..8d6c0da 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelper.java
@@ -39,15 +39,17 @@
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class LSMRTreeDataflowHelper extends AbstractLSMRTreeDataflowHelper {
+ private int[] btreeFields;
public LSMRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
List<IVirtualBufferCache> virtualBufferCaches, IBinaryComparatorFactory[] btreeComparatorFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- ILinearizeComparatorFactory linearizeCmpFactory) {
+ ILinearizeComparatorFactory linearizeCmpFactory, int[] btreeFields) {
super(opDesc, ctx, partition, virtualBufferCaches, btreeComparatorFactories, valueProviderFactories,
rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, linearizeCmpFactory);
+ this.btreeFields = btreeFields;
}
public LSMRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
@@ -56,10 +58,11 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- ILinearizeComparatorFactory linearizeCmpFactory) {
+ ILinearizeComparatorFactory linearizeCmpFactory, int[] btreeFields) {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, btreeComparatorFactories,
valueProviderFactories, rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler,
ioOpCallbackFactory, linearizeCmpFactory);
+ this.btreeFields = btreeFields;
}
@Override
@@ -73,7 +76,7 @@
return LSMRTreeUtils.createLSMTree(virtualBufferCaches, file, diskBufferCache, diskFileMapProvider,
typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory);
+ ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory, btreeFields);
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
index 7b54f19..beb6c20 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeDataflowHelperFactory.java
@@ -39,19 +39,21 @@
private final IPrimitiveValueProviderFactory[] valueProviderFactories;
private final RTreePolicyType rtreePolicyType;
private final ILinearizeComparatorFactory linearizeCmpFactory;
+ private final int[] btreeFields;
public LSMRTreeDataflowHelperFactory(IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, IBinaryComparatorFactory[] btreeComparatorFactories,
IVirtualBufferCacheProvider virtualBufferCacheProvider, ILSMMergePolicyFactory mergePolicyFactory,
Map<String, String> mergePolicyProperties, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- ILinearizeComparatorFactory linearizeCmpFactory, double bloomFilterFalsePositiveRate) {
+ ILinearizeComparatorFactory linearizeCmpFactory, double bloomFilterFalsePositiveRate, int[] btreeFields) {
super(virtualBufferCacheProvider, mergePolicyFactory, mergePolicyProperties, opTrackerFactory,
ioSchedulerProvider, ioOpCallbackFactory, bloomFilterFalsePositiveRate);
this.btreeComparatorFactories = btreeComparatorFactories;
this.valueProviderFactories = valueProviderFactories;
this.rtreePolicyType = rtreePolicyType;
this.linearizeCmpFactory = linearizeCmpFactory;
+ this.btreeFields = btreeFields;
}
@Override
@@ -61,6 +63,6 @@
virtualBufferCacheProvider.getVirtualBufferCaches(ctx), bloomFilterFalsePositiveRate,
btreeComparatorFactories, valueProviderFactories, rtreePolicyType,
mergePolicyFactory.createMergePolicy(mergePolicyProperties), opTrackerFactory,
- ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, linearizeCmpFactory);
+ ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, linearizeCmpFactory, btreeFields);
}
}
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index 673c7ae..7142929 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -97,8 +97,8 @@
BTree memBTree = new BTree(virtualBufferCache,
((IVirtualBufferCache) virtualBufferCache).getFileMapProvider(), new VirtualFreePageManager(
virtualBufferCache.getNumPages()), btreeInteriorFrameFactory, btreeLeafFrameFactory,
- btreeCmpFactories, fieldCount, new FileReference(new File(fileManager.getBaseDir() + "_virtual_b_"
- + i)));
+ btreeCmpFactories, btreeCmpFactories.length, new FileReference(new File(fileManager.getBaseDir()
+ + "_virtual_b_" + i)));
LSMRTreeMemoryComponent mutableComponent = new LSMRTreeMemoryComponent(memRTree, memBTree,
virtualBufferCache, i == 0 ? true : false);
memoryComponents.add(mutableComponent);
@@ -152,8 +152,7 @@
}
if (flushOnExit) {
- BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(
- ioOpCallback);
+ BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
accessor.scheduleFlush(cb);
try {
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 2be6d57..ffc92f2 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -31,6 +31,7 @@
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
@@ -39,9 +40,11 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.exceptions.TreeIndexDuplicateKeyException;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.tuples.DualTupleReference;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
@@ -63,6 +66,7 @@
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public class LSMRTree extends AbstractLSMRTree {
+ private int[] buddyBTreeFields;
public LSMRTree(List<IVirtualBufferCache> virtualBufferCaches, ITreeIndexFrameFactory rtreeInteriorFrameFactory,
ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory btreeInteriorFrameFactory,
@@ -72,13 +76,14 @@
IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, ILinearizeComparatorFactory linearizer,
int[] comparatorFields, IBinaryComparatorFactory[] linearizerArray, ILSMMergePolicy mergePolicy,
- ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback) {
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ int[] buddyBTreeFields) {
super(virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory,
btreeLeafFrameFactory, fileNameManager, new LSMRTreeDiskComponentFactory(diskRTreeFactory,
diskBTreeFactory, bloomFilterFactory), diskFileMapProvider, fieldCount, rtreeCmpFactories,
btreeCmpFactories, linearizer, comparatorFields, linearizerArray, bloomFilterFalsePositiveRate,
mergePolicy, opTracker, ioScheduler, ioOpCallback);
+ this.buddyBTreeFields = buddyBTreeFields;
}
/**
@@ -321,10 +326,11 @@
BTree btree = mergedComponent.getBTree();
IIndexBulkLoader btreeBulkLoader = btree.createBulkLoader(1.0f, true, 0L, false);
-
+
long numElements = 0L;
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
- numElements += ((LSMRTreeDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter().getNumElements();
+ numElements += ((LSMRTreeDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter()
+ .getNumElements();
}
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
@@ -332,7 +338,7 @@
bloomFilterFalsePositiveRate);
IIndexBulkLoader builder = mergedComponent.getBloomFilter().createBuilder(numElements,
bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
-
+
try {
while (btreeCursor.hasNext()) {
btreeCursor.next();
@@ -368,13 +374,29 @@
}
public class LSMRTreeAccessor extends LSMTreeIndexAccessor {
+ private DualTupleReference dualTuple = new DualTupleReference(buddyBTreeFields);
+
public LSMRTreeAccessor(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) {
super(lsmHarness, ctx);
}
@Override
public ITreeIndexCursor createSearchCursor(boolean exclusive) {
- return new LSMRTreeSearchCursor(ctx);
+ return new LSMRTreeSearchCursor(ctx, buddyBTreeFields);
+ }
+
+ @Override
+ public void delete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ dualTuple.reset(tuple);
+ ctx.setOperation(IndexOperation.DELETE);
+ lsmHarness.modify(ctx, false, dualTuple);
+ }
+
+ @Override
+ public void forceDelete(ITupleReference tuple) throws HyracksDataException, IndexException {
+ dualTuple.reset(tuple);
+ ctx.setOperation(IndexOperation.DELETE);
+ lsmHarness.forceModify(ctx, dualTuple);
}
public MultiComparator getMultiComparator() {
@@ -399,6 +421,30 @@
}
}
+ // This function is modified for R-Trees without antimatter tuples to allow buddy B-Tree to have only primary keys
+ @Override
+ public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException, IndexException {
+ LSMRTreeOpContext ctx = (LSMRTreeOpContext) ictx;
+ if (ctx.getOperation() == IndexOperation.PHYSICALDELETE) {
+ throw new UnsupportedOperationException("Physical delete not supported in the LSM-RTree");
+ }
+
+ ctx.modificationCallback.before(tuple);
+ ctx.modificationCallback.found(null, tuple);
+ if (ctx.getOperation() == IndexOperation.INSERT) {
+ ctx.currentMutableRTreeAccessor.insert(tuple);
+ } else {
+ // First remove all entries in the in-memory rtree (if any).
+ ctx.currentMutableRTreeAccessor.delete(tuple);
+ try {
+ ctx.currentMutableBTreeAccessor.insert(((DualTupleReference) tuple).getPermutingTuple());
+ } catch (TreeIndexDuplicateKeyException e) {
+ // Do nothing, because one delete tuple is enough to indicate
+ // that all the corresponding insert tuples are deleted
+ }
+ }
+ }
+
public class LSMRTreeBulkLoader implements IIndexBulkLoader {
private final ILSMComponent component;
private final IIndexBulkLoader bulkLoader;
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java
index f669585..ec9921c 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java
@@ -20,15 +20,18 @@
import edu.uci.ics.hyracks.storage.am.common.api.ICursorInitialState;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
+import edu.uci.ics.hyracks.storage.am.common.tuples.PermutingTupleReference;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
public class LSMRTreeSearchCursor extends LSMRTreeAbstractCursor {
private int currentCursor;
+ private PermutingTupleReference btreeTuple;
- public LSMRTreeSearchCursor(ILSMIndexOperationContext opCtx) {
+ public LSMRTreeSearchCursor(ILSMIndexOperationContext opCtx, int[] buddyBTreeFields) {
super(opCtx);
currentCursor = 0;
+ this.btreeTuple = new PermutingTupleReference(buddyBTreeFields);
}
@Override
@@ -77,12 +80,12 @@
while (rtreeCursors[currentCursor].hasNext()) {
rtreeCursors[currentCursor].next();
ITupleReference currentTuple = rtreeCursors[currentCursor].getTuple();
-
+ btreeTuple.reset(rtreeCursors[currentCursor].getTuple());
boolean killerTupleFound = false;
for (int i = 0; i < currentCursor; i++) {
btreeCursors[i].reset();
- btreeRangePredicate.setHighKey(currentTuple, true);
- btreeRangePredicate.setLowKey(currentTuple, true);
+ btreeRangePredicate.setHighKey(btreeTuple, true);
+ btreeRangePredicate.setLowKey(btreeTuple, true);
btreeAccessors[i].search(btreeCursors[i], btreeRangePredicate);
try {
if (btreeCursors[i].hasNext()) {
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
index ee942e7..13ecc84 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
@@ -66,9 +66,16 @@
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
- ILinearizeComparatorFactory linearizeCmpFactory) throws TreeIndexException {
+ ILinearizeComparatorFactory linearizeCmpFactory, int[] buddyBTreeFields) throws TreeIndexException {
+
+ ITypeTraits[] btreeTypeTraits = new ITypeTraits[buddyBTreeFields.length];
+ for (int i = 0; i < btreeTypeTraits.length; i++) {
+ btreeTypeTraits[i] = typeTraits[buddyBTreeFields[i]];
+ }
+
LSMTypeAwareTupleWriterFactory rtreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
- LSMTypeAwareTupleWriterFactory btreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, true);
+ LSMTypeAwareTupleWriterFactory btreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(btreeTypeTraits,
+ true);
ITreeIndexFrameFactory rtreeInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(rtreeTupleWriterFactory,
valueProviderFactories, rtreePolicyType);
@@ -87,7 +94,7 @@
typeTraits.length);
TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
freePageManagerFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmpFactories,
- typeTraits.length);
+ btreeTypeTraits.length);
int[] comparatorFields = { 0 };
IBinaryComparatorFactory[] linearizerArray = { linearizeCmpFactory };
@@ -105,7 +112,7 @@
btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory,
bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length,
rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory, comparatorFields, linearizerArray,
- mergePolicy, opTracker, ioScheduler, ioOpCallback);
+ mergePolicy, opTracker, ioScheduler, ioOpCallback, buddyBTreeFields);
return lsmTree;
}
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
index 6a519d0..ff3d21c 100644
--- a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeExamplesTest.java
@@ -52,12 +52,20 @@
@SuppressWarnings("rawtypes")
public abstract class AbstractRTreeExamplesTest {
+
+ public static enum RTreeType {
+ LSMRTREE,
+ LSMRTREE_WITH_ANTIMATTER,
+ RTREE
+ };
+
protected static final Logger LOGGER = Logger.getLogger(AbstractRTreeExamplesTest.class.getName());
protected final Random rnd = new Random(50);
+ protected RTreeType rTreeType;
protected abstract ITreeIndex createTreeIndex(ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
- IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType)
+ IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType, int[] btreeFields)
throws TreeIndexException;
/**
@@ -96,21 +104,37 @@
rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
// Declare BTree keys, this will only be used for LSMRTree
- int btreeKeyFieldCount = 6;
- IBinaryComparatorFactory[] btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
- btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[5] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ int btreeKeyFieldCount;
+ IBinaryComparatorFactory[] btreeCmpFactories;
+ int[] btreeFields = null;
+ if (rTreeType == RTreeType.LSMRTREE) {
+ //Parameters look different for LSM RTREE from LSM RTREE WITH ANTI MATTER TUPLES
+ btreeKeyFieldCount = 2;
+ btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeFields = new int[btreeKeyFieldCount];
+ for (int i = 0; i < btreeKeyFieldCount; i++) {
+ btreeFields[i] = rtreeKeyFieldCount + i;
+ }
+
+ } else {
+ btreeKeyFieldCount = 6;
+ btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[5] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ }
// create value providers
IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
rtreeCmpFactories.length, IntegerPointable.FACTORY);
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, RTreePolicyType.RTREE);
+ valueProviderFactories, RTreePolicyType.RTREE, btreeFields);
treeIndex.create();
treeIndex.activate();
@@ -192,20 +216,36 @@
rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
// Declare BTree keys, this will only be used for LSMRTree
- int btreeKeyFieldCount = 5;
- IBinaryComparatorFactory[] btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
- btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ int btreeKeyFieldCount;
+ IBinaryComparatorFactory[] btreeCmpFactories;
+ int[] btreeFields = null;
+ if (rTreeType == RTreeType.LSMRTREE) {
+ //Parameters look different for LSM RTREE from LSM RTREE WITH ANTI MATTER TUPLES
+ btreeKeyFieldCount = 1;
+ btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ btreeFields = new int[btreeKeyFieldCount];
+ for (int i = 0; i < btreeKeyFieldCount; i++) {
+ btreeFields[i] = rtreeKeyFieldCount + i;
+ }
+
+ } else {
+ btreeKeyFieldCount = 5;
+ btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ }
// create value providers
IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
rtreeCmpFactories.length, IntegerPointable.FACTORY);
+ //2
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, RTreePolicyType.RTREE);
+ valueProviderFactories, RTreePolicyType.RTREE, btreeFields);
treeIndex.create();
treeIndex.activate();
@@ -316,20 +356,35 @@
rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
// Declare BTree keys, this will only be used for LSMRTree
- int btreeKeyFieldCount = 5;
- IBinaryComparatorFactory[] btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
- btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ int btreeKeyFieldCount;
+ IBinaryComparatorFactory[] btreeCmpFactories;
+ int[] btreeFields = null;
+ if (rTreeType == RTreeType.LSMRTREE) {
+ //Parameters look different for LSM RTREE from LSM RTREE WITH ANTI MATTER TUPLES
+ btreeKeyFieldCount = 1;
+ btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ btreeFields = new int[btreeKeyFieldCount];
+ for (int i = 0; i < btreeKeyFieldCount; i++) {
+ btreeFields[i] = rtreeKeyFieldCount + i;
+ }
+
+ } else {
+ btreeKeyFieldCount = 5;
+ btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ }
// create value providers
IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
rtreeCmpFactories.length, IntegerPointable.FACTORY);
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, RTreePolicyType.RSTARTREE);
+ valueProviderFactories, RTreePolicyType.RSTARTREE, btreeFields);
treeIndex.create();
treeIndex.activate();
@@ -447,23 +502,39 @@
rtreeCmpFactories[4] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
rtreeCmpFactories[5] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
- // Declare RTree keys.
- int btreeKeyFieldCount = 7;
- IBinaryComparatorFactory[] btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
- btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
- btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
- btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
- btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
- btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
- btreeCmpFactories[5] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
- btreeCmpFactories[6] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ // Declare BTree keys, this will only be used for LSMRTree
+ int btreeKeyFieldCount;
+ IBinaryComparatorFactory[] btreeCmpFactories;
+ int[] btreeFields = null;
+ if (rTreeType == RTreeType.LSMRTREE) {
+ //Parameters look different for LSM RTREE from LSM RTREE WITH ANTI MATTER TUPLES
+ btreeKeyFieldCount = 1;
+ btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeFields = new int[btreeKeyFieldCount];
+ for (int i = 0; i < btreeKeyFieldCount; i++) {
+ btreeFields[i] = rtreeKeyFieldCount + i;
+ }
+
+ } else {
+ btreeKeyFieldCount = 7;
+ btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeCmpFactories[5] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ btreeCmpFactories[6] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ }
// create value providers
IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
rtreeCmpFactories.length, DoublePointable.FACTORY);
+ //4
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, RTreePolicyType.RTREE);
+ valueProviderFactories, RTreePolicyType.RTREE, btreeFields);
treeIndex.create();
treeIndex.activate();
@@ -541,21 +612,36 @@
rtreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- // Declare BTree keys.
- int btreeKeyFieldCount = 5;
- IBinaryComparatorFactory[] btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
- btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ // Declare BTree keys, this will only be used for LSMRTree
+ int btreeKeyFieldCount;
+ IBinaryComparatorFactory[] btreeCmpFactories;
+ int[] btreeFields = null;
+ if (rTreeType == RTreeType.LSMRTREE) {
+ //Parameters look different for LSM RTREE from LSM RTREE WITH ANTI MATTER TUPLES
+ btreeKeyFieldCount = 1;
+ btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeFields = new int[btreeKeyFieldCount];
+ for (int i = 0; i < btreeKeyFieldCount; i++) {
+ btreeFields[i] = rtreeKeyFieldCount + i;
+ }
+
+ } else {
+ btreeKeyFieldCount = 5;
+ btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ }
// create value providers
IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
rtreeCmpFactories.length, IntegerPointable.FACTORY);
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, RTreePolicyType.RTREE);
+ valueProviderFactories, RTreePolicyType.RTREE, btreeFields);
treeIndex.create();
treeIndex.activate();
@@ -662,21 +748,37 @@
rtreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- // Declare BTree keys.
- int btreeKeyFieldCount = 5;
- IBinaryComparatorFactory[] btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
- btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ // Declare BTree keys, this will only be used for LSMRTree
+ int btreeKeyFieldCount;
+ IBinaryComparatorFactory[] btreeCmpFactories;
+ int[] btreeFields = null;
+ if (rTreeType == RTreeType.LSMRTREE) {
+ //Parameters look different for LSM RTREE from LSM RTREE WITH ANTI MATTER TUPLES
+ btreeKeyFieldCount = 1;
+ btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeFields = new int[btreeKeyFieldCount];
+ for (int i = 0; i < btreeKeyFieldCount; i++) {
+ btreeFields[i] = rtreeKeyFieldCount + i;
+ }
+
+ } else {
+ btreeKeyFieldCount = 5;
+ btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
+ btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ }
// create value providers
IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
rtreeCmpFactories.length, IntegerPointable.FACTORY);
+ //6
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories,
- valueProviderFactories, RTreePolicyType.RTREE);
+ valueProviderFactories, RTreePolicyType.RTREE, btreeFields);
treeIndex.create();
treeIndex.activate();
diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeMultiThreadTest.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeMultiThreadTest.java
index d97a2d7..4aa24c6 100644
--- a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeMultiThreadTest.java
+++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeMultiThreadTest.java
@@ -38,6 +38,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.config.AccessMethodTestsConfig;
+import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeExamplesTest.RTreeType;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
@@ -45,9 +46,11 @@
public abstract class AbstractRTreeMultiThreadTest {
protected final boolean testRstarPolicy;
+ protected final RTreeType rTreeType;
- public AbstractRTreeMultiThreadTest(boolean testRstarPolicy) {
+ public AbstractRTreeMultiThreadTest(boolean testRstarPolicy, RTreeType rTreeType) {
this.testRstarPolicy = testRstarPolicy;
+ this.rTreeType = rTreeType;
}
protected final Logger LOGGER = Logger.getLogger(AbstractRTreeMultiThreadTest.class.getName());
@@ -66,7 +69,7 @@
protected abstract ITreeIndex createTreeIndex(ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories,
- IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType)
+ IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType, int[] btreeFields)
throws TreeIndexException;
protected abstract IIndexTestWorkerFactory getWorkerFactory();
@@ -89,11 +92,23 @@
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] rtreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeys);
- IBinaryComparatorFactory[] btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes,
- fieldSerdes.length);
+ int[] btreeFields = null;
+ IBinaryComparatorFactory[] btreeCmpFactories;
+ if (rTreeType == RTreeType.LSMRTREE) {
+ int numBtreeFields = fieldSerdes.length - numKeys;
+ ISerializerDeserializer[] btreeFieldSerdes = new ISerializerDeserializer[numBtreeFields];
+ btreeFields = new int[numBtreeFields];
+ for (int i = 0; i < numBtreeFields; i++) {
+ btreeFields[i] = numKeys + i;
+ btreeFieldSerdes[i] = fieldSerdes[numKeys + i];
+ }
+ btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(btreeFieldSerdes, numBtreeFields);
+ } else {
+ btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, fieldSerdes.length);
+ }
ITreeIndex index = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories,
- rtreePolicyType);
+ rtreePolicyType, btreeFields);
IIndexTestWorkerFactory workerFactory = getWorkerFactory();
// 4 batches per thread.
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
index dc55f5d..f174ce4 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
@@ -37,7 +37,7 @@
return LSMBTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, cmpFactories,
bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback());
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), true);
}
@Before
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
index 2234075..9881975 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
@@ -48,7 +48,7 @@
SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), bloomFilterKeyFields,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
NoOpOperationTrackerProvider.INSTANCE.getOperationTracker(null), harness.getIOScheduler(),
- harness.getIOOperationCallback());
+ harness.getIOOperationCallback(), true);
}
@Override
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
index afaf30b..d4a0857 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
@@ -55,7 +55,7 @@
SerdeUtils.serdesToComparatorFactories(keySerdes, keySerdes.length), bloomFilterKeyFields,
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
NoOpOperationTrackerProvider.INSTANCE.getOperationTracker(null), harness.getIOScheduler(),
- harness.getIOOperationCallback());
+ harness.getIOOperationCallback(), true);
}
@Override
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
index adee618..c6f0a65 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
@@ -53,7 +53,7 @@
return LSMBTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, cmpFactories,
bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
- harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback());
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), true);
}
@Override
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
index fb06a7e..045aed0 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
@@ -109,7 +109,7 @@
lsmtree = LSMBTreeUtils.createLSMTree(virtualBufferCaches, file, bufferCache, fmp, typeTraits, cmpFactories,
bloomFilterKeyFields, bloomFilterFalsePositiveRate, NoMergePolicy.INSTANCE,
- new ThreadCountingTracker(), ioScheduler, NoOpIOOperationCallback.INSTANCE);
+ new ThreadCountingTracker(), ioScheduler, NoOpIOOperationCallback.INSTANCE, true);
}
@Override
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
index 6f7c23a..84ea6df 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
@@ -76,7 +76,7 @@
}
LSMBTree lsmTree = LSMBTreeUtils.createLSMTree(virtualBufferCaches, file, diskBufferCache, diskFileMapProvider,
typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
- ioScheduler, ioOpCallback);
+ ioScheduler, ioOpCallback, true);
LSMBTreeTestContext testCtx = new LSMBTreeTestContext(fieldSerdes, lsmTree);
return testCtx;
}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
index b04ae16..2b02656 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeExamplesTest.java
@@ -33,16 +33,21 @@
public class LSMRTreeExamplesTest extends AbstractRTreeExamplesTest {
private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
+ public LSMRTreeExamplesTest() {
+ super();
+ this.rTreeType = RTreeType.LSMRTREE;
+ }
+
@Override
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType) throws TreeIndexException {
+ RTreePolicyType rtreePolicyType, int[] btreeFields) throws TreeIndexException {
return LSMRTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, rtreeCmpFactories,
btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
harness.getIOOperationCallback(),
- LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length));
+ LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), btreeFields);
}
@Before
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
index a51abef..3ce30c5 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesExamplesTest.java
@@ -33,10 +33,15 @@
public class LSMRTreeWithAntiMatterTuplesExamplesTest extends AbstractRTreeExamplesTest {
private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
+ public LSMRTreeWithAntiMatterTuplesExamplesTest() {
+ super();
+ this.rTreeType = RTreeType.LSMRTREE_WITH_ANTIMATTER;
+ }
+
@Override
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType) throws TreeIndexException {
+ RTreePolicyType rtreePolicyType, int[] btreeFields) throws TreeIndexException {
return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits,
rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
index 077bd45..c3ec1b3 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeMultiThreadTest.java
@@ -30,6 +30,7 @@
import edu.uci.ics.hyracks.storage.am.common.datagen.ProbabilityHelper;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestHarness;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.utils.LSMRTreeUtils;
+import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeExamplesTest.RTreeType;
import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeMultiThreadTest;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
@@ -40,7 +41,7 @@
private LSMRTreeTestWorkerFactory workerFactory = new LSMRTreeTestWorkerFactory();
public LSMRTreeMultiThreadTest() {
- super(false);
+ super(false, RTreeType.LSMRTREE);
}
@Override
@@ -56,13 +57,13 @@
@Override
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType) throws TreeIndexException {
+ RTreePolicyType rtreePolicyType, int[] btreeFields) throws TreeIndexException {
return LSMRTreeUtils.createLSMTree(harness.getVirtualBufferCaches(), harness.getFileReference(),
harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits, rtreeCmpFactories,
btreeCmpFactories, valueProviderFactories, rtreePolicyType, harness.getBoomFilterFalsePositiveRate(),
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
harness.getIOOperationCallback(),
- LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length));
+ LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), btreeFields);
}
@Override
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
index ef871dd..22a4b07 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesMultiThreadTest.java
@@ -30,6 +30,7 @@
import edu.uci.ics.hyracks.storage.am.common.datagen.ProbabilityHelper;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestHarness;
import edu.uci.ics.hyracks.storage.am.lsm.rtree.utils.LSMRTreeUtils;
+import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeExamplesTest.RTreeType;
import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeMultiThreadTest;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
@@ -40,7 +41,7 @@
private LSMRTreeWithAntiMatterTuplesTestWorkerFactory workerFactory = new LSMRTreeWithAntiMatterTuplesTestWorkerFactory();
public LSMRTreeWithAntiMatterTuplesMultiThreadTest() {
- super(false);
+ super(false, RTreeType.LSMRTREE_WITH_ANTIMATTER);
}
@Override
@@ -56,7 +57,7 @@
@Override
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType) throws TreeIndexException {
+ RTreePolicyType rtreePolicyType, int[] btreeFields) throws TreeIndexException {
return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(harness.getVirtualBufferCaches(),
harness.getFileReference(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), typeTraits,
rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
index 42e793b..484ebc6 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
@@ -75,12 +75,19 @@
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] rtreeCmpFactories = SerdeUtils
.serdesToComparatorFactories(fieldSerdes, numKeyFields);
- IBinaryComparatorFactory[] btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes,
- fieldSerdes.length);
+ int numBtreeFields = fieldSerdes.length - numKeyFields;
+ ISerializerDeserializer[] btreeFieldSerdes = new ISerializerDeserializer[numBtreeFields];
+ int[] btreeFields = new int[numBtreeFields];
+ for (int i = 0; i < numBtreeFields; i++) {
+ btreeFields[i] = numKeyFields + i;
+ btreeFieldSerdes[i] = fieldSerdes[numKeyFields + i];
+ }
+ IBinaryComparatorFactory[] btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(btreeFieldSerdes,
+ numBtreeFields);
LSMRTree lsmTree = LSMRTreeUtils.createLSMTree(virtualBufferCaches, file, diskBufferCache, diskFileMapProvider,
typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback,
- LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length));
+ LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), btreeFields);
LSMRTreeTestContext testCtx = new LSMRTreeTestContext(fieldSerdes, lsmTree);
return testCtx;
}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeExamplesTest.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeExamplesTest.java
index d260ee2..8bec6d9 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeExamplesTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeExamplesTest.java
@@ -31,6 +31,11 @@
public class RTreeExamplesTest extends AbstractRTreeExamplesTest {
private final RTreeTestHarness harness = new RTreeTestHarness();
+ public RTreeExamplesTest() {
+ super();
+ this.rTreeType = RTreeType.RTREE;
+ }
+
@Before
public void setUp() throws HyracksDataException {
harness.setUp();
@@ -44,7 +49,7 @@
@Override
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType) throws TreeIndexException {
+ RTreePolicyType rtreePolicyType, int[] btreeFields) throws TreeIndexException {
return RTreeUtils.createRTree(harness.getBufferCache(), harness.getFileMapProvider(), typeTraits,
valueProviderFactories, rtreeCmpFactories, rtreePolicyType, harness.getFileReference());
}
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java
index 0730bba..ba1113e 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/multithread/RTreeMultiThreadTest.java
@@ -27,6 +27,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.datagen.ProbabilityHelper;
+import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeExamplesTest.RTreeType;
import edu.uci.ics.hyracks.storage.am.rtree.AbstractRTreeMultiThreadTest;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
@@ -35,7 +36,7 @@
public class RTreeMultiThreadTest extends AbstractRTreeMultiThreadTest {
public RTreeMultiThreadTest() {
- super(true);
+ super(true, RTreeType.RTREE);
}
private RTreeTestHarness harness = new RTreeTestHarness();
@@ -55,7 +56,7 @@
@Override
protected ITreeIndex createTreeIndex(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
- RTreePolicyType rtreePolicyType) throws TreeIndexException {
+ RTreePolicyType rtreePolicyType, int[] btreeFields) throws TreeIndexException {
return RTreeUtils.createRTree(harness.getBufferCache(), harness.getFileMapProvider(), typeTraits,
valueProviderFactories, rtreeCmpFactories, rtreePolicyType, harness.getFileReference());
diff --git a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGen.java b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGen.java
index 163e476..bde94a9 100644
--- a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGen.java
+++ b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGen.java
@@ -227,7 +227,7 @@
@Override
public JobSpecification generateCreatingJob() throws HyracksException {
Class<? extends WritableComparable<?>> vertexIdClass = BspUtils.getVertexIndexClass(conf);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
ITypeTraits[] typeTraits = new ITypeTraits[2];
typeTraits[0] = new TypeTraits(false);
typeTraits[1] = new TypeTraits(false);
@@ -261,7 +261,7 @@
Class<? extends WritableComparable<?>> vertexIdClass = BspUtils.getVertexIndexClass(conf);
Class<? extends Writable> vertexClass = BspUtils.getVertexClass(conf);
int maxFrameLimit = (int) (((long) 512 * MB) / frameSize);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
IFileSplitProvider fileSplitProvider = getFileSplitProvider(jobId, PRIMARY_INDEX);
/**
@@ -325,7 +325,7 @@
public JobSpecification scanIndexPrintGraph(String nodeName, String path) throws HyracksException {
Class<? extends WritableComparable<?>> vertexIdClass = BspUtils.getVertexIndexClass(conf);
Class<? extends Writable> vertexClass = BspUtils.getVertexClass(conf);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
/**
* construct empty tuple operator
@@ -465,7 +465,7 @@
* generate a "clear state" job
*/
public JobSpecification generateClearState() throws HyracksException {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
ClearStateOperatorDescriptor clearState = new ClearStateOperatorDescriptor(spec, jobId);
setLocationConstraint(spec, clearState);
spec.addRoot(clearState);
@@ -479,7 +479,7 @@
* @throws HyracksException
*/
protected JobSpecification dropIndex(String indexName) throws HyracksException {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
IFileSplitProvider fileSplitProvider = getFileSplitProvider(jobId, indexName);
IndexDropOperatorDescriptor drop = new IndexDropOperatorDescriptor(spec, storageManagerInterface,
@@ -507,7 +507,8 @@
if (BspUtils.useLSM(conf)) {
return new LSMBTreeDataflowHelperFactory(new VirtualBufferCacheProvider(),
new ConstantMergePolicyFactory(), MERGE_POLICY_PROPERTIES, NoOpOperationTrackerProvider.INSTANCE,
- SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE, 0.01);
+ /* TODO verify whether key dup check is required or not in preglix: to be safe, just check it as it has been done*/
+ SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE, 0.01, true);
} else {
return new BTreeDataflowHelperFactory();
}
@@ -518,7 +519,7 @@
Configuration conf = job.getConfiguration();
Class<? extends WritableComparable<?>> vertexIdClass = BspUtils.getVertexIndexClass(conf);
Class<? extends Writable> vertexClass = BspUtils.getVertexClass(conf);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
IFileSplitProvider fileSplitProvider = getFileSplitProvider(jobId, PRIMARY_INDEX);
/**
@@ -585,7 +586,7 @@
HyracksException {
Class<? extends WritableComparable<?>> vertexIdClass = BspUtils.getVertexIndexClass(conf);
Class<? extends Writable> vertexClass = BspUtils.getVertexClass(conf);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
/**
* construct empty tuple operator
@@ -673,7 +674,7 @@
/** generate plan specific state checkpointing */
protected JobSpecification[] generateStateCheckpointing(int lastSuccessfulIteration) throws HyracksException {
Class<? extends WritableComparable<?>> vertexIdClass = BspUtils.getVertexIndexClass(conf);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
/**
* source aggregate
@@ -726,7 +727,7 @@
}
Configuration conf = tmpJob.getConfiguration();
Class vertexIdClass = BspUtils.getVertexIndexClass(conf);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
/***
* HDFS read operator
@@ -825,7 +826,7 @@
private JobSpecification bulkLoadLiveVertexBTree(int iteration) throws HyracksException {
Class<? extends WritableComparable<?>> vertexIdClass = BspUtils.getVertexIndexClass(conf);
Class<? extends Writable> vertexClass = BspUtils.getVertexClass(conf);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
/**
* construct empty tuple operator
@@ -889,7 +890,7 @@
public void setLocationConstraint(JobSpecification spec, IOperatorDescriptor operator) {
optimizer.setOptimizedLocationConstraints(spec, operator);
}
-
+
/**
* get the file split provider
*
@@ -897,7 +898,7 @@
* @param indexName
* @return the IFileSplitProvider instance
*/
- public IFileSplitProvider getFileSplitProvider(String jobId, String indexName){
+ public IFileSplitProvider getFileSplitProvider(String jobId, String indexName) {
return optimizer.getOptimizedFileSplitProvider(jobId, indexName);
}
diff --git a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenInnerJoin.java b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenInnerJoin.java
index f838fb8..a0da2a3 100644
--- a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenInnerJoin.java
+++ b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenInnerJoin.java
@@ -108,7 +108,7 @@
Class<? extends Writable> messageValueClass = BspUtils.getMessageValueClass(conf);
String[] partialAggregateValueClassNames = BspUtils.getPartialAggregateValueClassNames(conf);
IConfigurationFactory confFactory = new ConfigurationFactory(conf);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
/**
* construct empty tuple operator
@@ -311,7 +311,7 @@
Class<? extends Writable> vertexClass = BspUtils.getVertexClass(conf);
Class<? extends Writable> messageValueClass = BspUtils.getMessageValueClass(conf);
String[] partialAggregateValueClassNames = BspUtils.getPartialAggregateValueClassNames(conf);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
/**
* source aggregate
@@ -567,7 +567,7 @@
private JobSpecification generateSecondaryBTreeCheckpointLoad(int lastSuccessfulIteration, PregelixJob job)
throws HyracksException {
Class<? extends WritableComparable<?>> vertexIdClass = BspUtils.getVertexIndexClass(job.getConfiguration());
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
String checkpointPath = BspUtils.getSecondaryIndexCheckpointPath(conf, lastSuccessfulIteration);
PregelixJob tmpJob = createCloneJob("State checkpoint loading for job " + jobId, job);
@@ -650,7 +650,7 @@
Class<? extends Writable> msgListClass = MsgList.class;
String readFile = lastSuccessfulIteration % 2 == 0 ? SECONDARY_INDEX_EVEN : SECONDARY_INDEX_ODD;
IFileSplitProvider secondaryFileSplitProviderRead = getFileSplitProvider(jobId, readFile);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
/**
* construct empty tuple operator
*/
diff --git a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenOuterJoin.java b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenOuterJoin.java
index 39a56bf..2853fd0 100644
--- a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenOuterJoin.java
+++ b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenOuterJoin.java
@@ -84,7 +84,7 @@
Class<? extends Writable> messageValueClass = BspUtils.getMessageValueClass(conf);
String[] partialAggregateValueClassNames = BspUtils.getPartialAggregateValueClassNames(conf);
IConfigurationFactory confFactory = new ConfigurationFactory(conf);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
/**
* construct empty tuple operator
@@ -266,7 +266,7 @@
Class<? extends Writable> vertexClass = BspUtils.getVertexClass(conf);
Class<? extends Writable> messageValueClass = BspUtils.getMessageValueClass(conf);
String[] partialAggregateValueClassNames = BspUtils.getPartialAggregateValueClassNames(conf);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
/**
* source aggregate
diff --git a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenOuterJoinSingleSort.java b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenOuterJoinSingleSort.java
index c10e6c2..a72777b 100644
--- a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenOuterJoinSingleSort.java
+++ b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenOuterJoinSingleSort.java
@@ -84,7 +84,7 @@
Class<? extends Writable> messageValueClass = BspUtils.getMessageValueClass(conf);
String[] partialAggregateValueClassNames = BspUtils.getPartialAggregateValueClassNames(conf);
IConfigurationFactory confFactory = new ConfigurationFactory(conf);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
/**
* construct empty tuple operator
@@ -257,7 +257,7 @@
Class<? extends Writable> vertexClass = BspUtils.getVertexClass(conf);
Class<? extends Writable> messageValueClass = BspUtils.getMessageValueClass(conf);
String[] partialAggregateValueClassNames = BspUtils.getPartialAggregateValueClassNames(conf);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
/**
* source aggregate
diff --git a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenOuterJoinSort.java b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenOuterJoinSort.java
index 953d82c..e28b06b 100644
--- a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenOuterJoinSort.java
+++ b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGenOuterJoinSort.java
@@ -80,7 +80,7 @@
Class<? extends Writable> messageValueClass = BspUtils.getMessageValueClass(conf);
String[] partialAggregateValueClassNames = BspUtils.getPartialAggregateValueClassNames(conf);
IConfigurationFactory confFactory = new ConfigurationFactory(conf);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
/**
* construct empty tuple operator
@@ -267,7 +267,7 @@
Class<? extends Writable> vertexClass = BspUtils.getVertexClass(conf);
Class<? extends Writable> messageValueClass = BspUtils.getMessageValueClass(conf);
String[] partialAggregateValueClassNames = BspUtils.getPartialAggregateValueClassNames(conf);
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification(frameSize);
/**
* source aggregate