Merged 0.1.3_btreefix into trunk
git-svn-id: https://hyracks.googlecode.com/svn/trunk/hyracks@332 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeOperatorsTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeOperatorsTest.java
index 33aebc1..d3c047e 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeOperatorsTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeOperatorsTest.java
@@ -18,10 +18,10 @@
import java.io.DataOutput;
import java.io.File;
+import org.junit.Before;
import org.junit.Test;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
@@ -30,7 +30,6 @@
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
@@ -44,454 +43,554 @@
import edu.uci.ics.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.misc.NullSinkOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.sort.InMemorySortOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeCursor;
+import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeBulkLoadOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeInsertUpdateDeleteOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeRegistry;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.IBTreeRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.btree.frames.MetaDataFrame;
import edu.uci.ics.hyracks.storage.am.btree.frames.NSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.NSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOp;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
-import edu.uci.ics.hyracks.storage.am.btree.impls.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
-import edu.uci.ics.hyracks.storage.am.btree.impls.RangeSearchCursor;
import edu.uci.ics.hyracks.storage.am.btree.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.test.support.TestBTreeRegistryProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
import edu.uci.ics.hyracks.test.support.TestStorageManagerInterface;
import edu.uci.ics.hyracks.tests.integration.AbstractIntegrationTest;
public class BTreeOperatorsTest extends AbstractIntegrationTest {
- static {
- TestStorageManagerComponentHolder.init(8192, 20);
- }
+ static {
+ TestStorageManagerComponentHolder.init(8192, 20);
+ }
- @Test
- public void bulkLoadTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
+ private IBTreeRegistryProvider btreeRegistryProvider = new TestBTreeRegistryProvider();
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/orders-part1.tbl"))) };
- IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
- RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ private final String sep = System.getProperty("file.separator");
- FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(spec, ordersSplitProvider,
- new DelimitedDataTupleParserFactory(new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE }, '|'), ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID);
+ // field, type and key declarations for primary index
+ private int primaryFieldCount = 6;
+ private ITypeTrait[] primaryTypeTraits = new ITypeTrait[primaryFieldCount];
+ private int primaryKeyFieldCount = 1;
+ private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
+ private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(
+ primaryTypeTraits);
+ private IBTreeInteriorFrameFactory primaryInteriorFrameFactory = new NSMInteriorFrameFactory(
+ primaryTupleWriterFactory);
+ private IBTreeLeafFrameFactory primaryLeafFrameFactory = new NSMLeafFrameFactory(
+ primaryTupleWriterFactory);
- InMemorySortOperatorDescriptor sorter = new InMemorySortOperatorDescriptor(spec, new int[] { 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID);
+ private String primaryBtreeName = "primary.ix";
+ private String primaryNc1FileName = System.getProperty("java.io.tmpdir")
+ + sep + "nc1" + sep + primaryBtreeName;
- // declare fields
- int fieldCount = 3;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- typeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- typeTraits[2] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ private IFileSplitProvider primaryBtreeSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
+ primaryNc1FileName))) });
- // declare keys
- int keyFieldCount = 1;
- IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[keyFieldCount];
- comparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ private RecordDescriptor primaryRecDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- // SimpleTupleWriterFactory tupleWriterFactory = new
- // SimpleTupleWriterFactory();
- IBTreeInteriorFrameFactory interiorFrameFactory = new NSMInteriorFrameFactory(tupleWriterFactory);
- IBTreeLeafFrameFactory leafFrameFactory = new NSMLeafFrameFactory(tupleWriterFactory);
- IBTreeRegistryProvider btreeRegistryProvider = new TestBTreeRegistryProvider();
+ // field, type and key declarations for secondary indexes
+ private int secondaryFieldCount = 2;
+ private ITypeTrait[] secondaryTypeTraits = new ITypeTrait[secondaryFieldCount];
+ private int secondaryKeyFieldCount = 2;
+ private IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[secondaryKeyFieldCount];
+ private TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(
+ secondaryTypeTraits);
+ private IBTreeInteriorFrameFactory secondaryInteriorFrameFactory = new NSMInteriorFrameFactory(
+ secondaryTupleWriterFactory);
+ private IBTreeLeafFrameFactory secondaryLeafFrameFactory = new NSMLeafFrameFactory(
+ secondaryTupleWriterFactory);
- int[] fieldPermutation = { 0, 4, 5 };
- String btreeName = "btree.bin";
- FileReference nc1File = new FileReference(new File(System.getProperty("java.io.tmpdir") + "/nc1/" + btreeName));
- IFileSplitProvider btreeSplitProvider = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- nc1File) });
+ private String secondaryBtreeName = "secondary.ix";
+ private String secondaryNc1FileName = System.getProperty("java.io.tmpdir")
+ + sep + "nc1" + sep + secondaryBtreeName;
- IStorageManagerInterface storageManager = new TestStorageManagerInterface();
- BTreeBulkLoadOperatorDescriptor btreeBulkLoad = new BTreeBulkLoadOperatorDescriptor(spec, storageManager,
- btreeRegistryProvider, btreeSplitProvider, interiorFrameFactory, leafFrameFactory, typeTraits,
- comparatorFactories, fieldPermutation, 0.7f);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, btreeBulkLoad, NC1_ID);
+ private IFileSplitProvider secondaryBtreeSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
+ secondaryNc1FileName))) });
- spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
+ private RecordDescriptor secondaryRecDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0, btreeBulkLoad, 0);
+ @Before
+ public void setup() {
+ // field, type and key declarations for primary index
+ primaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ primaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ primaryTypeTraits[2] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ primaryTypeTraits[3] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ primaryTypeTraits[4] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ primaryTypeTraits[5] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ primaryComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
- spec.addRoot(btreeBulkLoad);
- runTest(spec);
+ // field, type and key declarations for secondary indexes
+ secondaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ secondaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ secondaryComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ secondaryComparatorFactories[1] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ }
- // construct a multicomparator from the factories (only for printing
- // purposes)
- IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
- for (int i = 0; i < comparatorFactories.length; i++) {
- comparators[i] = comparatorFactories[i].createBinaryComparator();
- }
+ @Test
+ public void loadPrimaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- MultiComparator cmp = new MultiComparator(typeTraits, comparators);
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID,
+ new FileReference(new File("data/tpch0.001/orders-part1.tbl"))) };
+ IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(
+ ordersSplits);
+ RecordDescriptor ordersDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- // try an ordered scan on the bulk-loaded btree
- int btreeFileId = storageManager.getFileMapProvider(null).lookupFileId(nc1File);
- storageManager.getBufferCache(null).openFile(btreeFileId);
- BTree btree = btreeRegistryProvider.getBTreeRegistry(null).get(btreeFileId);
- IBTreeCursor scanCursor = new RangeSearchCursor(leafFrameFactory.getFrame());
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- BTreeOpContext opCtx = btree.createOpContext(BTreeOp.BTO_SEARCH, leafFrameFactory.getFrame(),
- interiorFrameFactory.getFrame(), null);
- btree.search(scanCursor, nullPred, opCtx);
- try {
- while (scanCursor.hasNext()) {
- scanCursor.next();
- ITupleReference frameTuple = scanCursor.getTuple();
- String rec = cmp.printTuple(frameTuple, ordersDesc.getFields());
- System.out.println(rec);
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- scanCursor.close();
- }
- storageManager.getBufferCache(null).closeFile(btreeFileId);
- }
+ FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(
+ spec, ordersSplitProvider, new DelimitedDataTupleParserFactory(
+ new IValueParserFactory[] {
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE }, '|'),
+ ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ ordScanner, NC1_ID);
- @Test
- public void btreeSearchTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(
+ spec,
+ 1000,
+ new int[] { 0 },
+ new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
+ ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter,
+ NC1_ID);
- // declare fields
- int fieldCount = 3;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- typeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- typeTraits[2] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
+ BTreeBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new BTreeBulkLoadOperatorDescriptor(
+ spec, storageManager, btreeRegistryProvider,
+ primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits,
+ primaryComparatorFactories, fieldPermutation, 0.7f);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ primaryBtreeBulkLoad, NC1_ID);
- // declare keys
- int keyFieldCount = 1;
- IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[keyFieldCount];
- comparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0,
+ sorter, 0);
- TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- // SimpleTupleWriterFactory tupleWriterFactory = new
- // SimpleTupleWriterFactory();
- IBTreeInteriorFrameFactory interiorFrameFactory = new NSMInteriorFrameFactory(tupleWriterFactory);
- IBTreeLeafFrameFactory leafFrameFactory = new NSMLeafFrameFactory(tupleWriterFactory);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0,
+ primaryBtreeBulkLoad, 0);
- // construct a multicomparator from the factories (only for printing
- // purposes)
- IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
- for (int i = 0; i < comparatorFactories.length; i++) {
- comparators[i] = comparatorFactories[i].createBinaryComparator();
- }
- MultiComparator cmp = new MultiComparator(typeTraits, comparators);
+ spec.addRoot(primaryBtreeBulkLoad);
+ runTest(spec);
+ }
- // build tuple containing low and high search key
- ArrayTupleBuilder tb = new ArrayTupleBuilder(cmp.getKeyFieldCount() * 2); // high
- // key
- // and
- // low
- // key
- DataOutput dos = tb.getDataOutput();
+ @Test
+ public void scanPrimaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- tb.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize("100", dos); // low
- // key
- tb.addFieldEndOffset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize("200", dos); // high
- // key
- tb.addFieldEndOffset();
+ // build dummy tuple containing nothing
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
+ DataOutput dos = tb.getDataOutput();
- ISerializerDeserializer[] keyRecDescSers = { UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
- RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
+ tb.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("0", dos);
+ tb.addFieldEndOffset();
- ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
- keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID);
- IBTreeRegistryProvider btreeRegistryProvider = new TestBTreeRegistryProvider();
+ ISerializerDeserializer[] keyRecDescSers = {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- RecordDescriptor recDesc = new RecordDescriptor(new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
+ spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
+ tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ keyProviderOp, NC1_ID);
- String btreeName = "btree.bin";
- FileReference nc1File = new FileReference(new File(System.getProperty("java.io.tmpdir") + "/nc1/" + btreeName));
- IFileSplitProvider btreeSplitProvider = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- nc1File) });
+ int[] lowKeyFields = null; // - infinity
+ int[] highKeyFields = null; // + infinity
- IStorageManagerInterface storageManager = new TestStorageManagerInterface();
- BTreeSearchOperatorDescriptor btreeSearchOp = new BTreeSearchOperatorDescriptor(spec, recDesc, storageManager,
- btreeRegistryProvider, btreeSplitProvider, interiorFrameFactory, leafFrameFactory, typeTraits,
- comparatorFactories, true, new int[] { 0 }, new int[] { 1 }, true, true);
- // BTreeDiskOrderScanOperatorDescriptor btreeSearchOp = new
- // BTreeDiskOrderScanOperatorDescriptor(spec, splitProvider, recDesc,
- // bufferCacheProvider, btreeRegistryProvider, 0, "btreetest.bin",
- // interiorFrameFactory, leafFrameFactory, cmp);
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
+ spec, primaryRecDesc, storageManager, btreeRegistryProvider,
+ primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits,
+ primaryComparatorFactories, true, lowKeyFields, highKeyFields,
+ true, true);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ primaryBtreeSearchOp, NC1_ID);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, btreeSearchOp, NC1_ID);
+ PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
+ primaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec),
+ primaryBtreeSearchOp, 0, printer, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, btreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), btreeSearchOp, 0, printer, 0);
+ spec.addRoot(printer);
+ runTest(spec);
+ }
- spec.addRoot(printer);
- runTest(spec);
- }
+ @Test
+ public void searchPrimaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- @Test
- public void insertTest() throws Exception {
- // relies on the fact that NCs are run from same process
- System.setProperty("NodeControllerDataPath", System.getProperty("java.io.tmpdir") + "/");
+ // build tuple containing low and high search key
+ // high key and low key
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
+ DataOutput dos = tb.getDataOutput();
- JobSpecification spec = new JobSpecification();
+ tb.reset();
+ // low key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("100", dos);
+ tb.addFieldEndOffset();
+ // high key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("200", dos);
+ tb.addFieldEndOffset();
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/orders-part1.tbl"))) };
- IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
- RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ ISerializerDeserializer[] keyRecDescSers = {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(spec, ordersSplitProvider,
- new DelimitedDataTupleParserFactory(new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE }, '|'), ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
+ spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
+ tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ keyProviderOp, NC1_ID);
- // we will create a primary index and 2 secondary indexes
- // first create comparators for primary index
- int primaryFieldCount = 6;
- ITypeTrait[] primaryTypeTraits = new ITypeTrait[primaryFieldCount];
- primaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[2] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[3] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[4] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[5] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ int[] lowKeyFields = { 0 };
+ int[] highKeyFields = { 1 };
- int primaryKeyFieldCount = 1;
- IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- primaryComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
+ spec, primaryRecDesc, storageManager, btreeRegistryProvider,
+ primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits,
+ primaryComparatorFactories, true, lowKeyFields, highKeyFields,
+ true, true);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ primaryBtreeSearchOp, NC1_ID);
- TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
- // SimpleTupleWriterFactory primaryTupleWriterFactory = new
- // SimpleTupleWriterFactory();
- IBTreeInteriorFrameFactory primaryInteriorFrameFactory = new NSMInteriorFrameFactory(primaryTupleWriterFactory);
- IBTreeLeafFrameFactory primaryLeafFrameFactory = new NSMLeafFrameFactory(primaryTupleWriterFactory);
- IBTreeRegistryProvider btreeRegistryProvider = new TestBTreeRegistryProvider();
+ PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC1_ID);
- // construct a multicomparator for the primary index
- IBinaryComparator[] primaryComparators = new IBinaryComparator[primaryComparatorFactories.length];
- for (int i = 0; i < primaryComparatorFactories.length; i++) {
- primaryComparators[i] = primaryComparatorFactories[i].createBinaryComparator();
- }
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
+ primaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec),
+ primaryBtreeSearchOp, 0, printer, 0);
- MultiComparator primaryCmp = new MultiComparator(primaryTypeTraits, primaryComparators);
+ spec.addRoot(printer);
+ runTest(spec);
+ }
- // now create comparators for secondary indexes
- int secondaryFieldCount = 2;
- ITypeTrait[] secondaryTypeTraits = new ITypeTrait[secondaryFieldCount];
- secondaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- secondaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ @Test
+ public void loadSecondaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- int secondaryKeyFieldCount = 2;
- IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[secondaryKeyFieldCount];
- secondaryComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
- secondaryComparatorFactories[1] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ // build dummy tuple containing nothing
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
+ DataOutput dos = tb.getDataOutput();
- TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(secondaryTypeTraits);
- // SimpleTupleWriterFactory secondaryTupleWriterFactory = new
- // SimpleTupleWriterFactory();
- IBTreeInteriorFrameFactory secondaryInteriorFrameFactory = new NSMInteriorFrameFactory(
- secondaryTupleWriterFactory);
- IBTreeLeafFrameFactory secondaryLeafFrameFactory = new NSMLeafFrameFactory(secondaryTupleWriterFactory);
+ tb.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("0", dos);
+ tb.addFieldEndOffset();
- // construct a multicomparator for the secondary indexes
- IBinaryComparator[] secondaryComparators = new IBinaryComparator[secondaryComparatorFactories.length];
- for (int i = 0; i < secondaryComparatorFactories.length; i++) {
- secondaryComparators[i] = secondaryComparatorFactories[i].createBinaryComparator();
- }
+ ISerializerDeserializer[] keyRecDescSers = {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- MultiComparator secondaryCmp = new MultiComparator(secondaryTypeTraits, secondaryComparators);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
+ spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
+ tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ keyProviderOp, NC1_ID);
- // we create and register 3 btrees for in an insert pipeline being fed
- // from a filescan op
- BTreeRegistry btreeRegistry = btreeRegistryProvider.getBTreeRegistry(null);
- IStorageManagerInterface storageManager = new TestStorageManagerInterface();
- IBufferCache bufferCache = storageManager.getBufferCache(null);
- IFileMapProvider fileMapProvider = storageManager.getFileMapProvider(null);
+ int[] lowKeyFields = null; // - infinity
+ int[] highKeyFields = null; // + infinity
- // primary index
- FileReference fileA = new FileReference(new File("/tmp/btreetestA.ix"));
- bufferCache.createFile(fileA);
- int fileIdA = fileMapProvider.lookupFileId(fileA);
- bufferCache.openFile(fileIdA);
- BTree btreeA = new BTree(bufferCache, primaryInteriorFrameFactory, primaryLeafFrameFactory, primaryCmp);
- btreeA.create(fileIdA, primaryLeafFrameFactory.getFrame(), new MetaDataFrame());
- btreeA.open(fileIdA);
- btreeRegistry.register(fileIdA, btreeA);
- bufferCache.closeFile(fileIdA);
+ // scan primary index
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
+ spec, primaryRecDesc, storageManager, btreeRegistryProvider,
+ primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits,
+ primaryComparatorFactories, true, lowKeyFields, highKeyFields,
+ true, true);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ primaryBtreeSearchOp, NC1_ID);
- // first secondary index
- FileReference fileB = new FileReference(new File("/tmp/btreetestB.ix"));
- bufferCache.createFile(fileB);
- int fileIdB = fileMapProvider.lookupFileId(fileB);
- bufferCache.openFile(fileIdB);
- BTree btreeB = new BTree(bufferCache, secondaryInteriorFrameFactory, secondaryLeafFrameFactory, secondaryCmp);
- btreeB.create(fileIdB, secondaryLeafFrameFactory.getFrame(), new MetaDataFrame());
- btreeB.open(fileIdB);
- btreeRegistry.register(fileIdB, btreeB);
- bufferCache.closeFile(fileIdB);
+ // sort based on secondary keys
+ ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(
+ spec,
+ 1000,
+ new int[] { 3, 0 },
+ new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
+ primaryRecDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter,
+ NC1_ID);
- // second secondary index
- FileReference fileC = new FileReference(new File("/tmp/btreetestC.ix"));
- bufferCache.createFile(fileC);
- int fileIdC = fileMapProvider.lookupFileId(fileC);
- bufferCache.openFile(fileIdC);
- BTree btreeC = new BTree(bufferCache, secondaryInteriorFrameFactory, secondaryLeafFrameFactory, secondaryCmp);
- btreeC.create(fileIdC, secondaryLeafFrameFactory.getFrame(), new MetaDataFrame());
- btreeC.open(fileIdC);
- btreeRegistry.register(fileIdC, btreeC);
- bufferCache.closeFile(fileIdC);
+ // load secondary index
+ int[] fieldPermutation = { 3, 0 };
+ BTreeBulkLoadOperatorDescriptor secondaryBtreeBulkLoad = new BTreeBulkLoadOperatorDescriptor(
+ spec, storageManager, btreeRegistryProvider,
+ secondaryBtreeSplitProvider, secondaryInteriorFrameFactory,
+ secondaryLeafFrameFactory, secondaryTypeTraits,
+ secondaryComparatorFactories, fieldPermutation, 0.7f);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ secondaryBtreeBulkLoad, NC1_ID);
- // create insert operators
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
+ primaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec),
+ primaryBtreeSearchOp, 0, sorter, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0,
+ secondaryBtreeBulkLoad, 0);
- // primary index
- IFileSplitProvider btreeSplitProviderA = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- new FileReference(new File("/tmp/btreetestA.ix"))) });
- int[] fieldPermutationA = { 0, 1, 2, 3, 4, 5 };
- BTreeInsertUpdateDeleteOperatorDescriptor insertOpA = new BTreeInsertUpdateDeleteOperatorDescriptor(spec,
- ordersDesc, storageManager, btreeRegistryProvider, btreeSplitProviderA, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutationA,
- BTreeOp.BTO_INSERT);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, insertOpA, NC1_ID);
+ spec.addRoot(secondaryBtreeBulkLoad);
+ runTest(spec);
+ }
- // first secondary index
- IFileSplitProvider btreeSplitProviderB = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- new FileReference(new File("/tmp/btreetestB.ix"))) });
- int[] fieldPermutationB = { 3, 0 };
- BTreeInsertUpdateDeleteOperatorDescriptor insertOpB = new BTreeInsertUpdateDeleteOperatorDescriptor(spec,
- ordersDesc, storageManager, btreeRegistryProvider, btreeSplitProviderB, secondaryInteriorFrameFactory,
- secondaryLeafFrameFactory, secondaryTypeTraits, secondaryComparatorFactories, fieldPermutationB,
- BTreeOp.BTO_INSERT);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, insertOpB, NC1_ID);
+ @Test
+ public void searchSecondaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- // second secondary index
- IFileSplitProvider btreeSplitProviderC = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
- new FileReference(new File("/tmp/btreetestC.ix"))) });
- int[] fieldPermutationC = { 4, 0 };
- BTreeInsertUpdateDeleteOperatorDescriptor insertOpC = new BTreeInsertUpdateDeleteOperatorDescriptor(spec,
- ordersDesc, storageManager, btreeRegistryProvider, btreeSplitProviderC, secondaryInteriorFrameFactory,
- secondaryLeafFrameFactory, secondaryTypeTraits, secondaryComparatorFactories, fieldPermutationC,
- BTreeOp.BTO_INSERT);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, insertOpC, NC1_ID);
+ // build tuple containing search keys (only use the first key as search
+ // key)
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(secondaryKeyFieldCount);
+ DataOutput dos = tb.getDataOutput();
- NullSinkOperatorDescriptor nullSink = new NullSinkOperatorDescriptor(spec);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, nullSink, NC1_ID);
+ tb.reset();
+ // low key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("1998-07-21", dos);
+ tb.addFieldEndOffset();
+ // high key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("2000-10-18", dos);
+ tb.addFieldEndOffset();
- spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, insertOpA, 0);
+ ISerializerDeserializer[] keyRecDescSers = {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- spec.connect(new OneToOneConnectorDescriptor(spec), insertOpA, 0, insertOpB, 0);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
+ spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
+ tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ keyProviderOp, NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), insertOpB, 0, insertOpC, 0);
+ int[] secondaryLowKeyFields = { 0 };
+ int[] secondaryHighKeyFields = { 1 };
- spec.connect(new OneToOneConnectorDescriptor(spec), insertOpC, 0, nullSink, 0);
+ // search secondary index
+ BTreeSearchOperatorDescriptor secondaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
+ spec, secondaryRecDesc, storageManager, btreeRegistryProvider,
+ secondaryBtreeSplitProvider, secondaryInteriorFrameFactory,
+ secondaryLeafFrameFactory, secondaryTypeTraits,
+ secondaryComparatorFactories, true, secondaryLowKeyFields,
+ secondaryHighKeyFields, true, true);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ secondaryBtreeSearchOp, NC1_ID);
- spec.addRoot(nullSink);
- runTest(spec);
+ int[] primaryLowKeyFields = { 1 }; // second field from the tuples
+ // coming from secondary index
+ int[] primaryHighKeyFields = { 1 }; // second field from the tuples
+ // coming from secondary index
- // scan primary index
- System.out.println("PRINTING PRIMARY INDEX");
- bufferCache.openFile(fileIdA);
- IBTreeCursor scanCursorA = new RangeSearchCursor(primaryLeafFrameFactory.getFrame());
- RangePredicate nullPredA = new RangePredicate(true, null, null, true, true, null, null);
- BTreeOpContext opCtxA = btreeA.createOpContext(BTreeOp.BTO_SEARCH, primaryLeafFrameFactory.getFrame(),
- primaryInteriorFrameFactory.getFrame(), null);
- btreeA.search(scanCursorA, nullPredA, opCtxA);
- try {
- while (scanCursorA.hasNext()) {
- scanCursorA.next();
- ITupleReference frameTuple = scanCursorA.getTuple();
- String rec = primaryCmp.printTuple(frameTuple, ordersDesc.getFields());
- System.out.println(rec);
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- scanCursorA.close();
- }
- bufferCache.closeFile(fileIdA);
- System.out.println();
+ // search primary index
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
+ spec, primaryRecDesc, storageManager, btreeRegistryProvider,
+ primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits,
+ primaryComparatorFactories, true, primaryLowKeyFields,
+ primaryHighKeyFields, true, true);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ primaryBtreeSearchOp, NC1_ID);
- // scan first secondary index
- System.out.println("PRINTING FIRST SECONDARY INDEX");
- bufferCache.openFile(fileIdB);
- IBTreeCursor scanCursorB = new RangeSearchCursor(secondaryLeafFrameFactory.getFrame());
- RangePredicate nullPredB = new RangePredicate(true, null, null, true, true, null, null);
- BTreeOpContext opCtxB = btreeB.createOpContext(BTreeOp.BTO_SEARCH, secondaryLeafFrameFactory.getFrame(),
- secondaryInteriorFrameFactory.getFrame(), null);
- btreeB.search(scanCursorB, nullPredB, opCtxB);
- try {
- while (scanCursorB.hasNext()) {
- scanCursorB.next();
- ITupleReference frameTuple = scanCursorB.getTuple();
- String rec = secondaryCmp.printTuple(frameTuple, ordersDesc.getFields());
- System.out.println(rec);
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- scanCursorB.close();
- }
- bufferCache.closeFile(fileIdB);
- System.out.println();
+ PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC1_ID);
- // scan second secondary index
- System.out.println("PRINTING SECOND SECONDARY INDEX");
- bufferCache.openFile(fileIdC);
- IBTreeCursor scanCursorC = new RangeSearchCursor(secondaryLeafFrameFactory.getFrame());
- RangePredicate nullPredC = new RangePredicate(true, null, null, true, true, null, null);
- BTreeOpContext opCtxC = btreeC.createOpContext(BTreeOp.BTO_SEARCH, secondaryLeafFrameFactory.getFrame(),
- secondaryInteriorFrameFactory.getFrame(), null);
- btreeC.search(scanCursorC, nullPredC, opCtxC);
- try {
- while (scanCursorC.hasNext()) {
- scanCursorC.next();
- ITupleReference frameTuple = scanCursorC.getTuple();
- String rec = secondaryCmp.printTuple(frameTuple, ordersDesc.getFields());
- System.out.println(rec);
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- scanCursorC.close();
- }
- bufferCache.closeFile(fileIdC);
- System.out.println();
- }
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
+ secondaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec),
+ secondaryBtreeSearchOp, 0, primaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec),
+ primaryBtreeSearchOp, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void insertPipelineTest() throws Exception {
+
+ JobSpecification spec = new JobSpecification();
+
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID,
+ new FileReference(new File("data/tpch0.001/orders-part2.tbl"))) };
+ IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(
+ ordersSplits);
+ RecordDescriptor ordersDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
+
+ FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(
+ spec, ordersSplitProvider, new DelimitedDataTupleParserFactory(
+ new IValueParserFactory[] {
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE }, '|'),
+ ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ ordScanner, NC1_ID);
+
+ // insert into primary index
+ int[] primaryFieldPermutation = { 0, 1, 2, 4, 5, 7 };
+ BTreeInsertUpdateDeleteOperatorDescriptor primaryBtreeInsertOp = new BTreeInsertUpdateDeleteOperatorDescriptor(
+ spec, ordersDesc, storageManager, btreeRegistryProvider,
+ primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits,
+ primaryComparatorFactories, primaryFieldPermutation,
+ BTreeOp.BTO_INSERT);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ primaryBtreeInsertOp, NC1_ID);
+
+ // first secondary index
+ int[] fieldPermutationB = { 4, 0 };
+ BTreeInsertUpdateDeleteOperatorDescriptor secondaryInsertOp = new BTreeInsertUpdateDeleteOperatorDescriptor(
+ spec, ordersDesc, storageManager, btreeRegistryProvider,
+ secondaryBtreeSplitProvider, secondaryInteriorFrameFactory,
+ secondaryLeafFrameFactory, secondaryTypeTraits,
+ secondaryComparatorFactories, fieldPermutationB,
+ BTreeOp.BTO_INSERT);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ secondaryInsertOp, NC1_ID);
+
+ NullSinkOperatorDescriptor nullSink = new NullSinkOperatorDescriptor(
+ spec);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, nullSink,
+ NC1_ID);
+
+ spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0,
+ primaryBtreeInsertOp, 0);
+
+ spec.connect(new OneToOneConnectorDescriptor(spec),
+ primaryBtreeInsertOp, 0, secondaryInsertOp, 0);
+
+ spec.connect(new OneToOneConnectorDescriptor(spec), secondaryInsertOp,
+ 0, nullSink, 0);
+
+ spec.addRoot(nullSink);
+ runTest(spec);
+ }
+
+ @Test
+ public void searchUpdatedSecondaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ // build tuple containing search keys (only use the first key as search
+ // key)
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(secondaryKeyFieldCount);
+ DataOutput dos = tb.getDataOutput();
+
+ tb.reset();
+ // low key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("1998-07-21", dos);
+ tb.addFieldEndOffset();
+ // high key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("2000-10-18", dos);
+ tb.addFieldEndOffset();
+
+ ISerializerDeserializer[] keyRecDescSers = {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
+
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
+ spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
+ tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ keyProviderOp, NC1_ID);
+
+ int[] secondaryLowKeyFields = { 0 };
+ int[] secondaryHighKeyFields = { 1 };
+
+ // search secondary index
+ BTreeSearchOperatorDescriptor secondaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
+ spec, secondaryRecDesc, storageManager, btreeRegistryProvider,
+ secondaryBtreeSplitProvider, secondaryInteriorFrameFactory,
+ secondaryLeafFrameFactory, secondaryTypeTraits,
+ secondaryComparatorFactories, true, secondaryLowKeyFields,
+ secondaryHighKeyFields, true, true);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ secondaryBtreeSearchOp, NC1_ID);
+
+ // second field from the tuples coming from secondary index
+ int[] primaryLowKeyFields = { 1 };
+ // second field from the tuples coming from secondary index
+ int[] primaryHighKeyFields = { 1 };
+
+ // search primary index
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
+ spec, primaryRecDesc, storageManager, btreeRegistryProvider,
+ primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits,
+ primaryComparatorFactories, true, primaryLowKeyFields,
+ primaryHighKeyFields, true, true);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ primaryBtreeSearchOp, NC1_ID);
+
+ PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC1_ID);
+
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
+ secondaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec),
+ secondaryBtreeSearchOp, 0, primaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec),
+ primaryBtreeSearchOp, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
index eeee17d..5911ae9 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
@@ -174,7 +174,7 @@
throw new HyracksDataException(e);
}
}
-
+
@Override
public void close() throws HyracksDataException {
try {
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/RangeSearchCursor.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/RangeSearchCursor.java
index 433cc6a..2ef1905 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/RangeSearchCursor.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/RangeSearchCursor.java
@@ -60,8 +60,10 @@
if (page != null) {
page.releaseReadLatch();
bufferCache.unpin(page);
- page = null;
}
+ tupleIndex = 0;
+ page = null;
+ pred = null;
}
public ITupleReference getTuple() {
@@ -217,9 +219,11 @@
@Override
public void reset() {
- tupleIndex = 0;
- page = null;
- pred = null;
+ try {
+ close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
}
@Override