Added LSMBTree multi bulk-load test. Each bulk load creates a new on-disk component.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_lsm_tree@1091 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/IOrderedIndexTestContext.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/IOrderedIndexTestContext.java
index 78a860a..fb92ed3 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/IOrderedIndexTestContext.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/IOrderedIndexTestContext.java
@@ -42,9 +42,11 @@
public IBinaryComparator[] getComparators();
- public void insertIntCheckTuple(int[] fieldValues);
+ public CheckTuple createIntCheckTuple(int[] fieldValues);
- public void insertStringCheckTuple(String[] fieldValues);
+ public CheckTuple createStringCheckTuple(String[] fieldValues);
+
+ public void insertCheckTuple(CheckTuple checkTuple, TreeSet<CheckTuple> checkTuples);
public TreeSet<CheckTuple> getCheckTuples();
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexBulkLoadTest.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexBulkLoadTest.java
index 105008e..5d64f2d 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexBulkLoadTest.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexBulkLoadTest.java
@@ -24,8 +24,11 @@
@SuppressWarnings("rawtypes")
public abstract class OrderedIndexBulkLoadTest extends OrderedIndexTestDriver {
- public OrderedIndexBulkLoadTest(BTreeLeafFrameType[] leafFrameTypesToTest) {
+ private final int bulkLoadRounds;
+
+ public OrderedIndexBulkLoadTest(BTreeLeafFrameType[] leafFrameTypesToTest, int bulkLoadRounds) {
super(leafFrameTypesToTest);
+ this.bulkLoadRounds = bulkLoadRounds;
}
@Override
@@ -33,21 +36,21 @@
ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey)
throws Exception {
IOrderedIndexTestContext ctx = createTestContext(fieldSerdes, numKeys, leafType);
-
- // We assume all fieldSerdes are of the same type. Check the first one
- // to determine which field types to generate.
- if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
- OrderedIndexTestUtils.bulkLoadIntTuples(ctx, numTuplesToInsert, getRandom());
- } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
- OrderedIndexTestUtils.bulkLoadStringTuples(ctx, numTuplesToInsert, getRandom());
- }
-
- OrderedIndexTestUtils.checkPointSearches(ctx);
- OrderedIndexTestUtils.checkOrderedScan(ctx);
- OrderedIndexTestUtils.checkDiskOrderScan(ctx);
- OrderedIndexTestUtils.checkRangeSearch(ctx, lowKey, highKey, true, true);
- if (prefixLowKey != null && prefixHighKey != null) {
- OrderedIndexTestUtils.checkRangeSearch(ctx, prefixLowKey, prefixHighKey, true, true);
+ for (int i = 0; i < bulkLoadRounds; i++) {
+ // We assume all fieldSerdes are of the same type. Check the first one
+ // to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ OrderedIndexTestUtils.bulkLoadIntTuples(ctx, numTuplesToInsert, getRandom());
+ } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
+ OrderedIndexTestUtils.bulkLoadStringTuples(ctx, numTuplesToInsert, getRandom());
+ }
+ OrderedIndexTestUtils.checkPointSearches(ctx);
+ OrderedIndexTestUtils.checkOrderedScan(ctx);
+ OrderedIndexTestUtils.checkDiskOrderScan(ctx);
+ OrderedIndexTestUtils.checkRangeSearch(ctx, lowKey, highKey, true, true);
+ if (prefixLowKey != null && prefixHighKey != null) {
+ OrderedIndexTestUtils.checkRangeSearch(ctx, prefixLowKey, prefixHighKey, true, true);
+ }
}
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestContext.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestContext.java
index a56ed9c..888a5a9 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestContext.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestContext.java
@@ -60,20 +60,25 @@
}
@Override
- public void insertIntCheckTuple(int[] fieldValues) {
+ public CheckTuple createIntCheckTuple(int[] fieldValues) {
CheckTuple<Integer> checkTuple = new CheckTuple<Integer>(getFieldCount(), getKeyFieldCount());
for(int v : fieldValues) {
checkTuple.add(v);
}
- checkTuples.add(checkTuple);
+ return checkTuple;
}
@Override
- public void insertStringCheckTuple(String[] fieldValues) {
+ public CheckTuple createStringCheckTuple(String[] fieldValues) {
CheckTuple<String> checkTuple = new CheckTuple<String>(getFieldCount(), getKeyFieldCount());
for(String s : fieldValues) {
checkTuple.add((String)s);
}
+ return checkTuple;
+ }
+
+ @Override
+ public void insertCheckTuple(CheckTuple checkTuple, TreeSet<CheckTuple> checkTuples) {
checkTuples.add(checkTuple);
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestUtils.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestUtils.java
index 262a13e..e46f0e6 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestUtils.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/tests/OrderedIndexTestUtils.java
@@ -278,7 +278,7 @@
try {
ctx.getIndexAccessor().insert(ctx.getTuple());
// Set expected values. Do this only after insertion succeeds because we ignore duplicate keys.
- ctx.insertIntCheckTuple(fieldValues);
+ ctx.insertCheckTuple(ctx.createIntCheckTuple(fieldValues), ctx.getCheckTuples());
} catch (BTreeDuplicateKeyException e) {
// Ignore duplicate key insertions.
}
@@ -308,7 +308,7 @@
try {
ctx.getIndexAccessor().insert(ctx.getTuple());
// Set expected values. Do this only after insertion succeeds because we ignore duplicate keys.
- ctx.insertStringCheckTuple(fieldValues);
+ ctx.insertCheckTuple(ctx.createStringCheckTuple(fieldValues), ctx.getCheckTuples());
} catch (BTreeDuplicateKeyException e) {
// Ignore duplicate key insertions.
}
@@ -320,6 +320,7 @@
int numKeyFields = ctx.getKeyFieldCount();
int[] fieldValues = new int[ctx.getFieldCount()];
int maxValue = (int)Math.ceil(Math.pow(numTuples, 1.0/(double)numKeyFields));
+ TreeSet<CheckTuple> tmpCheckTuples = new TreeSet<CheckTuple>();
for (int i = 0; i < numTuples; i++) {
// Set keys.
for (int j = 0; j < numKeyFields; j++) {
@@ -331,15 +332,21 @@
}
// Set expected values. We also use these as the pre-sorted stream for bulk loading.
- ctx.insertIntCheckTuple(fieldValues);
+ ctx.insertCheckTuple(ctx.createIntCheckTuple(fieldValues), tmpCheckTuples);
}
- bulkLoadCheckTuples(ctx, numTuples);
+ bulkLoadCheckTuples(ctx, tmpCheckTuples);
+
+ // Add tmpCheckTuples to ctx check tuples for comparing searches.
+ for (CheckTuple checkTuple : tmpCheckTuples) {
+ ctx.insertCheckTuple(checkTuple, ctx.getCheckTuples());
+ }
}
public static void bulkLoadStringTuples(IOrderedIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
int fieldCount = ctx.getFieldCount();
int numKeyFields = ctx.getKeyFieldCount();
String[] fieldValues = new String[fieldCount];
+ TreeSet<CheckTuple> tmpCheckTuples = new TreeSet<CheckTuple>();
for (int i = 0; i < numTuples; i++) {
// Set keys.
for (int j = 0; j < numKeyFields; j++) {
@@ -351,19 +358,25 @@
fieldValues[j] = getRandomString(5, rnd);
}
// Set expected values. We also use these as the pre-sorted stream for bulk loading.
- ctx.insertStringCheckTuple(fieldValues);
+ ctx.insertCheckTuple(ctx.createStringCheckTuple(fieldValues), tmpCheckTuples);
}
- bulkLoadCheckTuples(ctx, numTuples);
+ bulkLoadCheckTuples(ctx, tmpCheckTuples);
+
+ // Add tmpCheckTuples to ctx check tuples for comparing searches.
+ for (CheckTuple checkTuple : tmpCheckTuples) {
+ ctx.insertCheckTuple(checkTuple, ctx.getCheckTuples());
+ }
}
- private static void bulkLoadCheckTuples(IOrderedIndexTestContext ctx, int numTuples) throws HyracksDataException, TreeIndexException {
+ private static void bulkLoadCheckTuples(IOrderedIndexTestContext ctx, TreeSet<CheckTuple> checkTuples) throws HyracksDataException, TreeIndexException {
int fieldCount = ctx.getFieldCount();
+ int numTuples = checkTuples.size();
ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
// Perform bulk load.
IIndexBulkLoadContext bulkLoadCtx = ctx.getIndex().beginBulkLoad(0.7f);
int c = 1;
- for (CheckTuple checkTuple : ctx.getCheckTuples()) {
+ for (CheckTuple checkTuple : checkTuples) {
if (LOGGER.isLoggable(Level.INFO)) {
if (c % (numTuples / 10) == 0) {
LOGGER.info("Bulk Loading Tuple " + c + "/" + numTuples);
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeBulkLoadTest.java
index f509284..3cf0d86 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeBulkLoadTest.java
@@ -32,7 +32,7 @@
public class BTreeBulkLoadTest extends OrderedIndexBulkLoadTest {
public BTreeBulkLoadTest() {
- super(BTreeTestHarness.LEAF_FRAMES_TO_TEST);
+ super(BTreeTestHarness.LEAF_FRAMES_TO_TEST, 1);
}
private final BTreeTestHarness harness = new BTreeTestHarness();
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
index 8103191..e1af5ad 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeBulkLoadTest.java
@@ -32,7 +32,7 @@
public class LSMBTreeBulkLoadTest extends OrderedIndexBulkLoadTest {
public LSMBTreeBulkLoadTest() {
- super(LSMBTreeTestHarness.LEAF_FRAMES_TO_TEST);
+ super(LSMBTreeTestHarness.LEAF_FRAMES_TO_TEST, 1);
}
private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
new file mode 100644
index 0000000..bb232c7
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/LSMBTreeMultiBulkLoadTest.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.lsm.btree;
+
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.tests.IOrderedIndexTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.tests.OrderedIndexBulkLoadTest;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
+
+@SuppressWarnings("rawtypes")
+public class LSMBTreeMultiBulkLoadTest extends OrderedIndexBulkLoadTest {
+ public LSMBTreeMultiBulkLoadTest() {
+ // Using 5 bulk load rounds.
+ super(LSMBTreeTestHarness.LEAF_FRAMES_TO_TEST, 5);
+ }
+
+ private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected IOrderedIndexTestContext createTestContext(ISerializerDeserializer[] fieldSerdes, int numKeys,
+ BTreeLeafFrameType leafType) throws Exception {
+ return LSMBTreeTestContext.create(harness.getMemBufferCache(), harness.getMemFreePageManager(),
+ harness.getOnDiskDir(), harness.getDiskBufferCache(), harness.getDiskFileMapProvider(), fieldSerdes,
+ numKeys, harness.getFileId());
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
index 91cd677..22e8373 100644
--- a/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
+++ b/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
@@ -15,6 +15,8 @@
package edu.uci.ics.hyracks.storage.am.lsm.btree.util;
+import java.util.TreeSet;
+
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
@@ -52,26 +54,7 @@
* Override to provide upsert semantics for the check tuples.
*/
@Override
- public void insertIntCheckTuple(int[] fieldValues) {
- CheckTuple<Integer> checkTuple = new CheckTuple<Integer>(getFieldCount(), getKeyFieldCount());
- for(int v : fieldValues) {
- checkTuple.add(v);
- }
- if (checkTuples.contains(checkTuple)) {
- checkTuples.remove(checkTuple);
- }
- checkTuples.add(checkTuple);
- }
-
- /**
- * Override to provide upsert semantics for the check tuples.
- */
- @Override
- public void insertStringCheckTuple(String[] fieldValues) {
- CheckTuple<String> checkTuple = new CheckTuple<String>(getFieldCount(), getKeyFieldCount());
- for(String s : fieldValues) {
- checkTuple.add((String)s);
- }
+ public void insertCheckTuple(CheckTuple checkTuple, TreeSet<CheckTuple> checkTuples) {
if (checkTuples.contains(checkTuple)) {
checkTuples.remove(checkTuple);
}